From 36c54ed1a2fcb9e186ee4bce2581ce37db544be1 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Tue, 17 May 2022 22:44:37 +0300 Subject: [PATCH 01/19] implemented oauth support --- .../connectors/source-bing-ads/README.md | 3 +- .../integration_tests/invalid_config.json | 31 +-- .../source-bing-ads/source_bing_ads/client.py | 28 ++- .../source-bing-ads/source_bing_ads/spec.json | 205 +++++++++++++----- .../oauth/OAuthImplementationFactory.java | 1 + .../flows/MicrosoftBingAdsOAuthFlow.java | 77 +++++++ 6 files changed, 270 insertions(+), 75 deletions(-) create mode 100644 airbyte-oauth/src/main/java/io/airbyte/oauth/flows/MicrosoftBingAdsOAuthFlow.java diff --git a/airbyte-integrations/connectors/source-bing-ads/README.md b/airbyte-integrations/connectors/source-bing-ads/README.md index 598e2be036cc..22a0fc0fa7f1 100644 --- a/airbyte-integrations/connectors/source-bing-ads/README.md +++ b/airbyte-integrations/connectors/source-bing-ads/README.md @@ -116,7 +116,8 @@ Customize `acceptance-test-config.yml` file to configure tests. See [Source Acce If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. To run your integration tests with acceptance tests, from the connector root, run ``` -python -m pytest integration_tests -p integration_tests.acceptance +docker build . --no-cache -t airbyte/source-bing-ads:dev \ +&& python -m pytest -p source_acceptance_test.plugin ``` To run your integration tests with docker diff --git a/airbyte-integrations/connectors/source-bing-ads/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-bing-ads/integration_tests/invalid_config.json index af0ab21003f5..cd3e697fcfca 100644 --- a/airbyte-integrations/connectors/source-bing-ads/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-bing-ads/integration_tests/invalid_config.json @@ -1,16 +1,19 @@ { - "accounts": { "selection_strategy": "all" }, - "user_id": "2222", - "customer_id": "1111", - "developer_token": "asgag4gwag3", - "refresh_token": "as2Ggas23gsa236gasgaskjfhas7i8ygf78as7osa7gy87asg8as7tg6as", - "client_secret": "1234", - "client_id": "123", - "tenant_id": "common", - "redirect_uri": "", - "reports_start_date": "2018-11-13", - "hourly_reports": true, - "daily_reports": false, - "weekly_reports": false, - "monthly_reports": true + "accounts": { "selection_strategy": "all" }, + "user_id": "2222", + "customer_id": "1111", + "developer_token": "asgag4gwag3", + "credentials": { + "auth_method": "oauth2.0", + "refresh_token": "as2Ggas23gsa236gasgaskjfhas7i8ygf78as7osa7gy87asg8as7tg6as", + "client_secret": "1234", + "client_id": "123" + }, + "reports_start_date": "2018-11-13", + "hourly_reports": false, + "daily_reports": false, + "weekly_reports": true, + "monthly_reports": true, + "tenant_id": "common", + "redirect_uri": "" } diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py index 441a144617fd..97f7b8f5931d 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py @@ -36,13 +36,11 @@ class Client: def __init__( self, + credentials: dict, developer_token: str, customer_id: str, - client_secret: str, - client_id: str, tenant_id: str, redirect_uri: str, - refresh_token: str, reports_start_date: str, hourly_reports: bool, daily_reports: bool, @@ -51,14 +49,8 @@ def __init__( **kwargs: Mapping[str, Any], ) -> None: self.authorization_data: Mapping[str, AuthorizationData] = {} - self.authentication = OAuthWebAuthCodeGrant( - client_id, - client_secret, - redirect_uri, - tenant=tenant_id, - ) - - self.refresh_token = refresh_token + self.authentication = self._get_auth_client(credentials, redirect_uri, tenant_id) + self.refresh_token = credentials["refresh_token"] self.customer_id = customer_id self.developer_token = developer_token self.hourly_reports = hourly_reports @@ -69,6 +61,20 @@ def __init__( self.oauth: OAuthTokens = self._get_access_token() self.reports_start_date = pendulum.parse(reports_start_date).astimezone(tz=timezone.utc) + def _get_auth_client(self, credentials: dict, redirect_uri: str, tenant_id: str) -> OAuthWebAuthCodeGrant: + # https://github.com/BingAds/BingAds-Python-SDK/blob/e7b5a618e87a43d0a5e2c79d9aa4626e208797bd/bingads/authorization.py#L390 + auth_creds = { + "client_id": credentials["client_id"], + # the `client_secret` should be provided for `non-public clients` ONLY + # https://docs.microsoft.com/en-us/advertising/guides/authentication-oauth-get-tokens?view=bingads-13#request-accesstoken + "client_secret": None, + "redirection_uri": redirect_uri, + "tenant": tenant_id, + } + if credentials["auth_method"] == "private_client": + auth_creds["client_secret"] = credentials["client_secret"] + return OAuthWebAuthCodeGrant(**auth_creds) + @lru_cache(maxsize=None) def _get_auth_data(self, account_id: Optional[str] = None) -> AuthorizationData: return AuthorizationData( diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json index d8dba81babfb..6e83e42d65c4 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json @@ -5,12 +5,10 @@ "title": "Bing Ads Spec", "type": "object", "required": [ + "developer_token", + "credentials", "accounts", - "client_id", - "client_secret", "customer_id", - "developer_token", - "refresh_token", "user_id", "reports_start_date", "hourly_reports", @@ -18,8 +16,92 @@ "weekly_reports", "monthly_reports" ], - "additionalProperties": false, + "additionalProperties": true, "properties": { + "tenant_id": { + "type": "string", + "title": "Tenant ID", + "description": "The Tenant ID of your Microsoft Advertising developer application. Set this to \"common\" unless you know you need a different value.", + "airbyte_secret": true, + "default": "common", + "order": 0 + }, + "credentials": { + "type": "object", + "title": "Authentication Method", + "description": "The authentication method to use to retrieve data from Microsoft Bings Ads", + "order": 1, + "oneOf": [ + { + "type": "object", + "title": "Microsoft OAuth2.0", + "description": "Microsoft API Credentials for connecting to Bings Ads", + "required": ["auth_method"], + "properties": { + "auth_method": { + "type": "string", + "const": "oauth2.0", + "order": 0 + }, + "client_id": { + "type": "string", + "title": "Client ID", + "description": "The Client ID of your Microsoft Advertising developer application.", + "airbyte_secret": true, + "order": 1 + }, + "client_secret": { + "type": "string", + "title": "Client Secret", + "description": "The Client Secret of your Microsoft Advertising developer application.", + "airbyte_secret": true, + "order": 2 + }, + "refresh_token": { + "type": "string", + "title": "Refresh Token", + "description": "Refresh Token to renew the expired Access Token.", + "airbyte_secret": true, + "order": 3 + } + } + }, + { + "type": "object", + "title": "Private OAuth2.0 Client", + "description": "Microsoft Private Client API Credentials for connecting to Bings Ads", + "required": ["auth_method", "client_id", "client_secret", "refresh_token"], + "properties": { + "auth_method": { + "type": "string", + "const": "private_client", + "order": 0 + }, + "client_id": { + "type": "string", + "title": "Client ID", + "description": "The Client ID of your Microsoft Advertising developer application.", + "airbyte_secret": true, + "order": 1 + }, + "client_secret": { + "type": "string", + "title": "Client Secret", + "description": "The Client Secret of your Microsoft Advertising developer application.", + "airbyte_secret": true, + "order": 2 + }, + "refresh_token": { + "type": "string", + "title": "Refresh Token", + "description": "Refresh Token to renew the expired Access Token.", + "airbyte_secret": true, + "order": 3 + } + } + } + ] + }, "accounts": { "title": "Accounts to replicate data from", "type": "object", @@ -59,71 +141,43 @@ } } } - ] - }, - "client_id": { - "type": "string", - "title": "Client ID", - "description": "The Client ID of your Microsoft Advertising developer application.", - "airbyte_secret": true, - "order": 0 - }, - "client_secret": { - "type": "string", - "title": "Client Secret", - "description": "The Client Secret of your Microsoft Advertising developer application.", - "airbyte_secret": true, - "order": 1 - }, - "refresh_token": { - "type": "string", - "title": "Refresh Token", - "description": "Refresh Token to renew the expired Access Token.", - "airbyte_secret": true, + ], "order": 2 }, "developer_token": { - "type": "string", - "title": "Developer Token", - "description": "Developer token associated with user.", - "airbyte_secret": true, - "order": 3 + "type": "string", + "title": "Developer Token", + "description": "Developer token associated with user.", + "airbyte_secret": true, + "order": 3 }, - "tenant_id": { + "customer_id": { "type": "string", - "title": "Tenant ID", - "description": "The Tenant ID of your Microsoft Advertising developer application. Set this to \"common\" unless you know you need a different value.", - "airbyte_secret": true, - "default": "common", + "title": "Customer ID", + "description": "Your Bing Customer ID. See the \"Getting Started\" section in the docs for information on how to obtain this ID", "order": 4 }, + "user_id": { + "type": "string", + "title": "User ID", + "description": "Bing Ads User ID. See the \"Getting Started\" section in the docs for information on how to obtain this ID", + "order": 5 + }, "redirect_uri": { "type": "string", "title": "Redirect URI (Optional)", "description": "The Redirect URI of your Microsoft Advertising developer application. Leave this empty unless you know that you need it.", "airbyte_secret": true, "default": "", - "order": 5 - }, - "customer_id": { - "type": "string", - "title": "Customer ID", - "description": "Your Bing Customer ID. See the \"Getting Started\" section in the docs for information on how to obtain this ID", "order": 6 }, - "user_id": { - "type": "string", - "title": "Account ID", - "description": "Bing Ads Account ID. See the \"Getting Started\" section in the docs for information on how to obtain this ID", - "order": 7 - }, "reports_start_date": { "type": "string", "title": "Reports replication start date", "format": "date", "default": "2020-01-01", "description": "The start date from which to begin replicating report data. Any data generated before this date will not be replicated in reports. This is a UTC date in YYYY-MM-DD format.", - "order": 8 + "order": 7 }, "hourly_reports": { "title": "Enable hourly-aggregate reports", @@ -150,5 +204,58 @@ "default": false } } + }, + "advanced_auth": { + "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_method"], + "predicate_value": "oauth2.0", + "oauth_config_specification": { + "complete_oauth_output_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "refresh_token": { + "type": "string", + "path_in_connector_config": ["credentials", "refresh_token"] + } + } + }, + "complete_oauth_server_input_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + } + } + }, + "complete_oauth_server_output_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "client_id": { + "type": "string", + "path_in_connector_config": ["credentials", "client_id"] + }, + "client_secret": { + "type": "string", + "path_in_connector_config": ["credentials", "client_secret"] + } + } + }, + "oauth_user_input_from_connector_config_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "tenant_id": { + "type": "string", + "path_in_connector_config": ["tenant_id"] + } + } + } + } } } diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java index 5d16b41cdc8d..309aa37d5c9d 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java @@ -35,6 +35,7 @@ public OAuthImplementationFactory(final ConfigRepository configRepository, final .put("airbyte/source-instagram", new InstagramOAuthFlow(configRepository, httpClient)) .put("airbyte/source-lever-hiring", new LeverOAuthFlow(configRepository, httpClient)) .put("airbyte/source-microsoft-teams", new MicrosoftTeamsOAuthFlow(configRepository, httpClient)) + .put("airbyte/source-bing-ads", new MicrosoftBingAdsOAuthFlow(configRepository, httpClient)) .put("airbyte/source-pipedrive", new PipeDriveOAuthFlow(configRepository, httpClient)) .put("airbyte/source-quickbooks", new QuickbooksOAuthFlow(configRepository, httpClient)) .put("airbyte/source-retently", new RetentlyOAuthFlow(configRepository, httpClient)) diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/MicrosoftBingAdsOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/MicrosoftBingAdsOAuthFlow.java new file mode 100644 index 000000000000..0a5c0688274f --- /dev/null +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/MicrosoftBingAdsOAuthFlow.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.oauth.flows; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.oauth.BaseOAuth2Flow; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.http.HttpClient; +import java.util.Map; +import java.util.UUID; +import java.util.function.Supplier; +import org.apache.http.client.utils.URIBuilder; + +public class MicrosoftBingAdsOAuthFlow extends BaseOAuth2Flow { + + public MicrosoftBingAdsOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient) { + super(configRepository, httpClient); + } + + public MicrosoftBingAdsOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient, final Supplier stateSupplier) { + super(configRepository, httpClient, stateSupplier); + } + + private String getScopes() { + return "offline_access%20https://ads.microsoft.com/msads.manage"; + } + + @Override + protected String formatConsentUrl(final UUID definitionId, + final String clientId, + final String redirectUrl, + final JsonNode inputOAuthConfiguration) + throws IOException { + + final String tenantId = getConfigValueUnsafe(inputOAuthConfiguration, "tenant_id"); + + try { + return new URIBuilder() + .setScheme("https") + .setHost("login.microsoftonline.com") + .setPath(tenantId + "/oauth2/v2.0/authorize") + .addParameter("client_id", clientId) + .addParameter("response_type", "code") + .addParameter("redirect_uri", redirectUrl) + .addParameter("response_mode", "query") + .addParameter("state", getState()) + .build().toString() + "&scope=" + getScopes(); + } catch (final URISyntaxException e) { + throw new IOException("Failed to format Consent URL for OAuth flow", e); + } + } + + @Override + protected Map getAccessTokenQueryParameters(final String clientId, + final String clientSecret, + final String authCode, + final String redirectUrl) { + return ImmutableMap.builder() + .put("client_id", clientId) + .put("code", authCode) + .put("redirect_uri", redirectUrl) + .put("grant_type", "authorization_code") + .build(); + } + + @Override + protected String getAccessTokenUrl(final JsonNode inputOAuthConfiguration) { + final String tenantId = getConfigValueUnsafe(inputOAuthConfiguration, "tenant_id"); + return "https://login.microsoftonline.com/" + tenantId + "/oauth2/v2.0/token"; + } + +} From 912de949e3e90f500ce13c87ef4d00c06dfbc2cd Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Tue, 17 May 2022 22:46:05 +0300 Subject: [PATCH 02/19] bumped version --- airbyte-integrations/connectors/source-bing-ads/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-bing-ads/Dockerfile b/airbyte-integrations/connectors/source-bing-ads/Dockerfile index 8349748f3000..683c8ee29044 100644 --- a/airbyte-integrations/connectors/source-bing-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-bing-ads/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.6 +LABEL io.airbyte.version=0.1.7 LABEL io.airbyte.name=airbyte/source-bing-ads From 5b908f03124c4d37594870b72ce17ef7b574777f Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Tue, 17 May 2022 23:06:01 +0300 Subject: [PATCH 03/19] added changelog --- .../connectors/source-bing-ads/source_bing_ads/client.py | 4 ++-- docs/integrations/sources/bing-ads.md | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py index 97f7b8f5931d..65d99f2bfcb2 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py @@ -65,13 +65,13 @@ def _get_auth_client(self, credentials: dict, redirect_uri: str, tenant_id: str) # https://github.com/BingAds/BingAds-Python-SDK/blob/e7b5a618e87a43d0a5e2c79d9aa4626e208797bd/bingads/authorization.py#L390 auth_creds = { "client_id": credentials["client_id"], - # the `client_secret` should be provided for `non-public clients` ONLY - # https://docs.microsoft.com/en-us/advertising/guides/authentication-oauth-get-tokens?view=bingads-13#request-accesstoken "client_secret": None, "redirection_uri": redirect_uri, "tenant": tenant_id, } if credentials["auth_method"] == "private_client": + # the `client_secret` should be provided for `non-public clients` ONLY + # https://docs.microsoft.com/en-us/advertising/guides/authentication-oauth-get-tokens?view=bingads-13#request-accesstoken auth_creds["client_secret"] = credentials["client_secret"] return OAuthWebAuthCodeGrant(**auth_creds) diff --git a/docs/integrations/sources/bing-ads.md b/docs/integrations/sources/bing-ads.md index 74461c5882b4..066fd166a030 100644 --- a/docs/integrations/sources/bing-ads.md +++ b/docs/integrations/sources/bing-ads.md @@ -66,6 +66,7 @@ API limits number of requests for all Microsoft Advertising clients. You can fin | Version | Date | Pull Request | Subject | |:--------| :--- |:---------------------------------------------------------| :--- | +| 0.1.7 | 2022-05-17 | [12937](https://github.com/airbytehq/airbyte/pull/12937) | Added OAuth2.0 authentication method | 0.1.6 | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500) | Improve input configuration copy | | 0.1.5 | 2022-01-01 | [11652](https://github.com/airbytehq/airbyte/pull/11652) | Rebump attempt after DockerHub failure at registring the 0.1.4 | | 0.1.4 | 2022-03-22 | [11311](https://github.com/airbytehq/airbyte/pull/11311) | Added optional Redirect URI & Tenant ID to spec | From 17c5b8ca297cf81cb9241d807ddbd9fdc7168351 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Tue, 17 May 2022 23:12:58 +0300 Subject: [PATCH 04/19] changed releaseStage to beta --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 575591d0f0e4..1940636f952f 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -108,7 +108,7 @@ documentationUrl: https://docs.airbyte.io/integrations/sources/bing-ads icon: bingads.svg sourceType: api - releaseStage: alpha + releaseStage: beta - name: Braintree sourceDefinitionId: 63cea06f-1c75-458d-88fe-ad48c7cb27fd dockerRepository: airbyte/source-braintree From c4edca2e8c537c4ed6e32ed493e3f6ae16d5dd61 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 18 May 2022 15:34:03 +0300 Subject: [PATCH 05/19] removed redirect_uri from spec --- .../connectors/source-bing-ads/source_bing_ads/client.py | 7 +++---- .../connectors/source-bing-ads/source_bing_ads/spec.json | 8 -------- docs/integrations/sources/bing-ads.md | 2 +- 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py index 65d99f2bfcb2..af88582d63d4 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py @@ -40,7 +40,6 @@ def __init__( developer_token: str, customer_id: str, tenant_id: str, - redirect_uri: str, reports_start_date: str, hourly_reports: bool, daily_reports: bool, @@ -49,7 +48,7 @@ def __init__( **kwargs: Mapping[str, Any], ) -> None: self.authorization_data: Mapping[str, AuthorizationData] = {} - self.authentication = self._get_auth_client(credentials, redirect_uri, tenant_id) + self.authentication = self._get_auth_client(credentials, tenant_id) self.refresh_token = credentials["refresh_token"] self.customer_id = customer_id self.developer_token = developer_token @@ -61,12 +60,12 @@ def __init__( self.oauth: OAuthTokens = self._get_access_token() self.reports_start_date = pendulum.parse(reports_start_date).astimezone(tz=timezone.utc) - def _get_auth_client(self, credentials: dict, redirect_uri: str, tenant_id: str) -> OAuthWebAuthCodeGrant: + def _get_auth_client(self, credentials: dict, tenant_id: str) -> OAuthWebAuthCodeGrant: # https://github.com/BingAds/BingAds-Python-SDK/blob/e7b5a618e87a43d0a5e2c79d9aa4626e208797bd/bingads/authorization.py#L390 auth_creds = { "client_id": credentials["client_id"], "client_secret": None, - "redirection_uri": redirect_uri, + "redirection_uri": "", # should be empty string "tenant": tenant_id, } if credentials["auth_method"] == "private_client": diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json index 6e83e42d65c4..d4b2999d6480 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json @@ -163,14 +163,6 @@ "description": "Bing Ads User ID. See the \"Getting Started\" section in the docs for information on how to obtain this ID", "order": 5 }, - "redirect_uri": { - "type": "string", - "title": "Redirect URI (Optional)", - "description": "The Redirect URI of your Microsoft Advertising developer application. Leave this empty unless you know that you need it.", - "airbyte_secret": true, - "default": "", - "order": 6 - }, "reports_start_date": { "type": "string", "title": "Reports replication start date", diff --git a/docs/integrations/sources/bing-ads.md b/docs/integrations/sources/bing-ads.md index 066fd166a030..d95139edefb4 100644 --- a/docs/integrations/sources/bing-ads.md +++ b/docs/integrations/sources/bing-ads.md @@ -66,7 +66,7 @@ API limits number of requests for all Microsoft Advertising clients. You can fin | Version | Date | Pull Request | Subject | |:--------| :--- |:---------------------------------------------------------| :--- | -| 0.1.7 | 2022-05-17 | [12937](https://github.com/airbytehq/airbyte/pull/12937) | Added OAuth2.0 authentication method +| 0.1.7 | 2022-05-17 | [12937](https://github.com/airbytehq/airbyte/pull/12937) | Added OAuth2.0 authentication method, removed `redirect_uri` from input configuration | 0.1.6 | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500) | Improve input configuration copy | | 0.1.5 | 2022-01-01 | [11652](https://github.com/airbytehq/airbyte/pull/11652) | Rebump attempt after DockerHub failure at registring the 0.1.4 | | 0.1.4 | 2022-03-22 | [11311](https://github.com/airbytehq/airbyte/pull/11311) | Added optional Redirect URI & Tenant ID to spec | From 569e67315d0b2360a7802f5a923451b202051583 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Thu, 19 May 2022 11:55:03 +0300 Subject: [PATCH 06/19] removed user_id, customer_id from spec --- .../connectors/source-bing-ads/setup.py | 2 +- .../source-bing-ads/source_bing_ads/client.py | 20 ++-- .../source_bing_ads/reports.py | 5 +- .../source-bing-ads/source_bing_ads/source.py | 78 ++++++++------- .../source-bing-ads/source_bing_ads/spec.json | 97 +++++-------------- docs/integrations/sources/bing-ads.md | 5 - 6 files changed, 75 insertions(+), 132 deletions(-) diff --git a/airbyte-integrations/connectors/source-bing-ads/setup.py b/airbyte-integrations/connectors/source-bing-ads/setup.py index 1c86a5ade1b3..897671c508de 100644 --- a/airbyte-integrations/connectors/source-bing-ads/setup.py +++ b/airbyte-integrations/connectors/source-bing-ads/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk", "bingads~=13.0.11", "vcrpy==4.1.1", "backoff==1.10.0", "pendulum==2.1.2"] +MAIN_REQUIREMENTS = ["airbyte-cdk", "bingads~=13.0.13", "vcrpy==4.1.1", "backoff==1.10.0", "pendulum==2.1.2"] TEST_REQUIREMENTS = [ "pytest~=6.1", diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py index af88582d63d4..1f8409ab2fb2 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py @@ -37,8 +37,6 @@ class Client: def __init__( self, credentials: dict, - developer_token: str, - customer_id: str, tenant_id: str, reports_start_date: str, hourly_reports: bool, @@ -50,8 +48,7 @@ def __init__( self.authorization_data: Mapping[str, AuthorizationData] = {} self.authentication = self._get_auth_client(credentials, tenant_id) self.refresh_token = credentials["refresh_token"] - self.customer_id = customer_id - self.developer_token = developer_token + self.developer_token = credentials["developer_token"] self.hourly_reports = hourly_reports self.daily_reports = daily_reports self.weekly_reports = weekly_reports @@ -75,10 +72,10 @@ def _get_auth_client(self, credentials: dict, tenant_id: str) -> OAuthWebAuthCod return OAuthWebAuthCodeGrant(**auth_creds) @lru_cache(maxsize=None) - def _get_auth_data(self, account_id: Optional[str] = None) -> AuthorizationData: + def _get_auth_data(self, customer_id: str = None, account_id: Optional[str] = None) -> AuthorizationData: return AuthorizationData( account_id=account_id, - customer_id=self.customer_id, + customer_id=customer_id, developer_token=self.developer_token, authentication=self.authentication, ) @@ -129,6 +126,7 @@ def _request( self, service_name: Optional[str], operation_name: str, + customer_id: Optional[str], account_id: Optional[str], params: Mapping[str, Any], is_report_service: bool = False, @@ -140,9 +138,9 @@ def _request( self.oauth = self._get_access_token() if is_report_service: - service = self._get_reporting_service(account_id=account_id) + service = self._get_reporting_service(customer_id=customer_id, account_id=account_id) else: - service = self.get_service(service_name=service_name, account_id=account_id) + service = self.get_service(service_name=service_name, customer_id=customer_id, account_id=account_id) return getattr(service, operation_name)(**params) @@ -150,22 +148,24 @@ def _request( def get_service( self, service_name: str, + customer_id: str = None, account_id: Optional[str] = None, ) -> ServiceClient: return ServiceClient( service=service_name, version=self.api_version, - authorization_data=self._get_auth_data(account_id), + authorization_data=self._get_auth_data(customer_id, account_id), environment=self.environment, ) @lru_cache(maxsize=None) def _get_reporting_service( self, + customer_id: Optional[str] = None, account_id: Optional[str] = None, ) -> ServiceClient: return ReportingServiceManager( - authorization_data=self._get_auth_data(account_id), + authorization_data=self._get_auth_data(customer_id, account_id), poll_interval_in_milliseconds=self.report_poll_interval, environment=self.environment, ) diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py index a872c66ff0a1..b6af1070d4c0 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py @@ -170,9 +170,10 @@ def get_updated_state( ) return current_stream_state - def send_request(self, params: Mapping[str, Any], account_id: str) -> _RowReport: + def send_request(self, params: Mapping[str, Any], customer_id: str, account_id: str) -> _RowReport: request_kwargs = { "service_name": None, + "customer_id": customer_id, "account_id": account_id, "operation_name": self.operation_name, "is_report_service": True, @@ -262,6 +263,6 @@ def stream_slices( **kwargs: Mapping[str, Any], ) -> Iterable[Optional[Mapping[str, Any]]]: for account in source_bing_ads.source.Accounts(self.client, self.config).read_records(SyncMode.full_refresh): - yield {"account_id": account["Id"]} + yield {"account_id": account["Id"], "customer_id": account["ParentCustomerId"]} yield from [] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py index eb6ae52d81bc..9298d47f257c 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py @@ -6,6 +6,8 @@ from abc import ABC, abstractmethod from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union +from requests import request + from airbyte_cdk import AirbyteLogger from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource @@ -13,6 +15,8 @@ from source_bing_ads.cache import VcrCache from source_bing_ads.client import Client from source_bing_ads.reports import ReportsMixin +from bingads.service_client import ServiceClient +from bingads.v13.reporting.reporting_service_manager import ReportingServiceManager from suds import sudsobject CACHE: VcrCache = VcrCache() @@ -61,6 +65,15 @@ def additional_fields(self) -> Optional[str]: """ pass + @property + def _service(self) -> Union[ServiceClient, ReportingServiceManager]: + return self.client.get_service(service_name=self.service_name) + + @property + def _user_id(self) -> int: + return self._service.GetUser().User.Id + + def next_page_token(self, response: sudsobject.Object, **kwargs: Mapping[str, Any]) -> Optional[Mapping[str, Any]]: """ Default method for streams that don't support pagination @@ -73,24 +86,21 @@ def parse_response(self, response: sudsobject.Object, **kwargs) -> Iterable[Mapp yield from [] - def send_request(self, params: Mapping[str, Any], account_id: str = None) -> Mapping[str, Any]: + def send_request(self, params: Mapping[str, Any], customer_id: str, account_id: str = None) -> Mapping[str, Any]: request_kwargs = { "service_name": self.service_name, + "customer_id": customer_id, "account_id": account_id, "operation_name": self.operation_name, "params": params, } - if not self.use_cache: - return self.client.request(**request_kwargs) - - with CACHE.use_cassette(): - return self.client.request(**request_kwargs) + request = self.client.request(**request_kwargs) + if self.use_cache: + with CACHE.use_cassette(): + return request + else: + return request - def get_account_id(self, stream_slice: Mapping[str, Any] = None) -> Optional[str]: - """ - Fetches account_id from slice object - """ - return str(stream_slice.get("account_id")) if stream_slice else None def read_records( self, @@ -101,14 +111,17 @@ def read_records( ) -> Iterable[Mapping[str, Any]]: stream_state = stream_state or {} next_page_token = None - account_id = self.get_account_id(stream_slice) + account_id = str(stream_slice.get("account_id")) if stream_slice else None + customer_id = str(stream_slice.get("customer_id")) if stream_slice else None while True: params = self.request_params( - stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token, account_id=account_id + stream_state=stream_state, + stream_slice=stream_slice, + next_page_token=next_page_token, + account_id=account_id, ) - - response = self.send_request(params, account_id=account_id) + response = self.send_request(params, customer_id=customer_id, account_id=account_id) for record in self.parse_response(response): yield record @@ -154,21 +167,12 @@ def request_params( { "Field": "UserId", "Operator": "Equals", - "Value": self.config["user_id"], + "Value": self._user_id, } ] } - if self.config["accounts"]["selection_strategy"] == "subset": - predicates["Predicate"].append( - { - "Field": "AccountId", - "Operator": "In", - "Value": ",".join(self.config["accounts"]["ids"]), - } - ) - - paging = self.client.get_service(service_name=self.service_name).factory.create("ns5:Paging") + paging = self._service.factory.create("ns5:Paging") paging.Index = next_page_token or 0 paging.Size = self.page_size_limit return { @@ -213,7 +217,7 @@ def stream_slices( **kwargs: Mapping[str, Any], ) -> Iterable[Optional[Mapping[str, Any]]]: for account in Accounts(self.client, self.config).read_records(SyncMode.full_refresh): - yield {"account_id": account["Id"]} + yield {"account_id": account["Id"], "customer_id": account["ParentCustomerId"]} yield from [] @@ -247,8 +251,8 @@ def stream_slices( ) -> Iterable[Optional[Mapping[str, Any]]]: campaigns = Campaigns(self.client, self.config) for account in Accounts(self.client, self.config).read_records(SyncMode.full_refresh): - for campaign in campaigns.read_records(sync_mode=SyncMode.full_refresh, stream_slice={"account_id": account["Id"]}): - yield {"campaign_id": campaign["Id"], "account_id": account["Id"]} + for campaign in campaigns.read_records(sync_mode=SyncMode.full_refresh, stream_slice={"account_id": account["Id"], "customer_id": account["ParentCustomerId"]}): + yield {"campaign_id": campaign["Id"], "account_id": account["Id"], "customer_id": account["ParentCustomerId"]} yield from [] @@ -294,7 +298,7 @@ def stream_slices( ad_groups = AdGroups(self.client, self.config) for slice in ad_groups.stream_slices(sync_mode=SyncMode.full_refresh): for ad_group in ad_groups.read_records(sync_mode=SyncMode.full_refresh, stream_slice=slice): - yield {"ad_group_id": ad_group["Id"], "account_id": slice["account_id"]} + yield {"ad_group_id": ad_group["Id"], "account_id": slice["account_id"], "customer_id": slice["customer_id"]} yield from [] @@ -570,20 +574,14 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> try: client = Client(**config) account_ids = {str(account["Id"]) for account in Accounts(client, config).read_records(SyncMode.full_refresh)} - - if config["accounts"]["selection_strategy"] == "subset": - config_account_ids = set(config["accounts"]["ids"]) - if not config_account_ids.issubset(account_ids): - raise Exception(f"Accounts with ids: {config_account_ids.difference(account_ids)} not found on this user.") - elif config["accounts"]["selection_strategy"] == "all": - if not account_ids: - raise Exception("You don't have accounts assigned to this user.") + if account_ids: + return True, None else: - raise Exception("Incorrect account selection strategy.") + raise Exception("You don't have accounts assigned to this user.") except Exception as error: return False, error - return True, None + def get_report_streams(self, aggregation_type: str) -> List[Stream]: return [ diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json index d4b2999d6480..28d2dddac378 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json @@ -5,11 +5,7 @@ "title": "Bing Ads Spec", "type": "object", "required": [ - "developer_token", "credentials", - "accounts", - "customer_id", - "user_id", "reports_start_date", "hourly_reports", "daily_reports", @@ -36,33 +32,40 @@ "type": "object", "title": "Microsoft OAuth2.0", "description": "Microsoft API Credentials for connecting to Bings Ads", - "required": ["auth_method"], + "required": ["auth_method", "developer_token"], "properties": { "auth_method": { "type": "string", "const": "oauth2.0", "order": 0 }, + "developer_token": { + "type": "string", + "title": "Developer Token", + "description": "Developer token associated with user. See more info in the docs.", + "airbyte_secret": true, + "order": 1 + }, "client_id": { "type": "string", "title": "Client ID", "description": "The Client ID of your Microsoft Advertising developer application.", "airbyte_secret": true, - "order": 1 + "order": 2 }, "client_secret": { "type": "string", "title": "Client Secret", "description": "The Client Secret of your Microsoft Advertising developer application.", "airbyte_secret": true, - "order": 2 + "order": 3 }, "refresh_token": { "type": "string", "title": "Refresh Token", "description": "Refresh Token to renew the expired Access Token.", "airbyte_secret": true, - "order": 3 + "order": 4 } } }, @@ -70,106 +73,52 @@ "type": "object", "title": "Private OAuth2.0 Client", "description": "Microsoft Private Client API Credentials for connecting to Bings Ads", - "required": ["auth_method", "client_id", "client_secret", "refresh_token"], + "required": ["auth_method", "developer_token", "client_id", "client_secret", "refresh_token"], "properties": { "auth_method": { "type": "string", "const": "private_client", "order": 0 }, + "developer_token": { + "type": "string", + "title": "Developer Token", + "description": "Developer token associated with user. See more info in the docs.", + "airbyte_secret": true, + "order": 1 + }, "client_id": { "type": "string", "title": "Client ID", "description": "The Client ID of your Microsoft Advertising developer application.", "airbyte_secret": true, - "order": 1 + "order": 2 }, "client_secret": { "type": "string", "title": "Client Secret", "description": "The Client Secret of your Microsoft Advertising developer application.", "airbyte_secret": true, - "order": 2 + "order": 3 }, "refresh_token": { "type": "string", "title": "Refresh Token", "description": "Refresh Token to renew the expired Access Token.", "airbyte_secret": true, - "order": 3 + "order": 4 } } } ] }, - "accounts": { - "title": "Accounts to replicate data from", - "type": "object", - "description": "", - "oneOf": [ - { - "title": "All Accounts", - "additionalProperties": false, - "description": "Replicate data from all accounts to which you have access.", - "required": ["selection_strategy"], - "properties": { - "selection_strategy": { - "type": "string", - "const": "all" - } - } - }, - { - "title": "Specific Accounts", - "additionalProperties": false, - "description": "Fetch data for subset of account IDs.", - "required": ["ids", "selection_strategy"], - "properties": { - "selection_strategy": { - "type": "string", - "const": "subset" - }, - "ids": { - "type": "array", - "title": "Account IDs", - "description": "List of the account IDs from which data will be replicated.", - "items": { - "type": "string" - }, - "minItems": 1, - "uniqueItems": true - } - } - } - ], - "order": 2 - }, - "developer_token": { - "type": "string", - "title": "Developer Token", - "description": "Developer token associated with user.", - "airbyte_secret": true, - "order": 3 - }, - "customer_id": { - "type": "string", - "title": "Customer ID", - "description": "Your Bing Customer ID. See the \"Getting Started\" section in the docs for information on how to obtain this ID", - "order": 4 - }, - "user_id": { - "type": "string", - "title": "User ID", - "description": "Bing Ads User ID. See the \"Getting Started\" section in the docs for information on how to obtain this ID", - "order": 5 - }, "reports_start_date": { "type": "string", "title": "Reports replication start date", "format": "date", "default": "2020-01-01", "description": "The start date from which to begin replicating report data. Any data generated before this date will not be replicated in reports. This is a UTC date in YYYY-MM-DD format.", - "order": 7 + "order": 2 }, "hourly_reports": { "title": "Enable hourly-aggregate reports", diff --git a/docs/integrations/sources/bing-ads.md b/docs/integrations/sources/bing-ads.md index d95139edefb4..0dc399c8fc5c 100644 --- a/docs/integrations/sources/bing-ads.md +++ b/docs/integrations/sources/bing-ads.md @@ -41,9 +41,6 @@ API limits number of requests for all Microsoft Advertising clients. You can fin ## Getting started (Airbyte Open Source) ### Requirements -* A Microsoft User account with access to at least one Microsoft Advertising account -* A Microsoft Ads Customer ID -* Your Microsoft User ID * A developer application with access to: * client ID * client secret @@ -56,8 +53,6 @@ API limits number of requests for all Microsoft Advertising clients. You can fin * Create a developer application using the instructions for [registering an application](https://docs.microsoft.com/en-us/advertising/guides/authentication-oauth-register?view=bingads-13) in Azure portal * Perform [these steps](https://docs.microsoft.com/en-us/advertising/guides/authentication-oauth-consent?view=bingads-13l) to get auth code, and use that to [get a refresh token](https://docs.microsoft.com/en-us/advertising/guides/authentication-oauth-get-tokens?view=bingads-13). For reference, the full authentication process described [here](https://docs.microsoft.com/en-us/advertising/guides/get-started?view=bingads-13#access-token). Be aware that the refresh token will expire in 90 days. You need to repeat the auth process to get a new refresh token. * Find your Microsoft developer token by following [these instructions](https://docs.microsoft.com/en-us/advertising/guides/get-started?view=bingads-13#get-developer-token) -* Find your customer ID and User ID by visiting the following URL: https://ui.ads.microsoft.com/campaign/Campaigns.m then copying the CID & UID parameters from the URL in the address bar. For example, once you visit the URL above, you'll notice it will have changed to an address of the form https://ui.ads.microsoft.com/campaign/vnext/overview?uid=USER_ID&cid=CUSTOMER_ID&aid=180534868 -- the customer ID is the value in the part of the URL that looks like `cid=THIS_IS_THE_CUSTOMER_ID&`, and the user ID is the value in front of `uid` e.g: `uid=THIS_IS_THE_USER_ID&`. -* Optionally, if you want to replicate data from specific ad account IDs (you can configure the Bing Ads connector to replicate data from all accounts you have access to, or only from some), then also grab the account IDs you want by visiting the [Accounts Summary](https://ui.ads.microsoft.com/campaign/vnext/accounts/performance) page, clicking on each of the accounts you want under the `Account name` column, then repeating the process described earlier to get the `aid` parameter in the URL that looks like `aid=ACCOUNT_ID&`. You'll need to do this process once for each account from which you want to replicate data. * Optionally, if your oauth app lives under a custom tenant which cannot use Microsoft's recommended `common` tenant, make sure to get the tenant ID ready for input when configuring the connector. The tenant will be used in the auth URL e.g: `https://login.microsoftonline.com//oauth2/v2.0/authorize`. From 0f2a28bab0b7e79f967b00cf91af062d859317d3 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Thu, 19 May 2022 12:00:49 +0300 Subject: [PATCH 07/19] fixed invalid_config --- .../integration_tests/invalid_config.json | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/airbyte-integrations/connectors/source-bing-ads/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-bing-ads/integration_tests/invalid_config.json index cd3e697fcfca..6cf658fbc36c 100644 --- a/airbyte-integrations/connectors/source-bing-ads/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-bing-ads/integration_tests/invalid_config.json @@ -1,10 +1,7 @@ -{ - "accounts": { "selection_strategy": "all" }, - "user_id": "2222", - "customer_id": "1111", - "developer_token": "asgag4gwag3", +{ "credentials": { "auth_method": "oauth2.0", + "developer_token": "asgag4gwag3", "refresh_token": "as2Ggas23gsa236gasgaskjfhas7i8ygf78as7osa7gy87asg8as7tg6as", "client_secret": "1234", "client_id": "123" @@ -14,6 +11,5 @@ "daily_reports": false, "weekly_reports": true, "monthly_reports": true, - "tenant_id": "common", - "redirect_uri": "" + "tenant_id": "common" } From dd075c01c19dbd6401269f6dadde140b16fffe40 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Thu, 19 May 2022 12:15:26 +0300 Subject: [PATCH 08/19] fixed flakeCheck --- .../source-bing-ads/source_bing_ads/client.py | 2 +- .../source-bing-ads/source_bing_ads/source.py | 22 ++++++++----------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py index 1f8409ab2fb2..01ccb57dc184 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py @@ -62,7 +62,7 @@ def _get_auth_client(self, credentials: dict, tenant_id: str) -> OAuthWebAuthCod auth_creds = { "client_id": credentials["client_id"], "client_secret": None, - "redirection_uri": "", # should be empty string + "redirection_uri": "", # should be empty string "tenant": tenant_id, } if credentials["auth_method"] == "private_client": diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py index 9298d47f257c..f8383a243938 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py @@ -6,17 +6,15 @@ from abc import ABC, abstractmethod from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union -from requests import request - from airbyte_cdk import AirbyteLogger from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream +from bingads.service_client import ServiceClient +from bingads.v13.reporting.reporting_service_manager import ReportingServiceManager from source_bing_ads.cache import VcrCache from source_bing_ads.client import Client from source_bing_ads.reports import ReportsMixin -from bingads.service_client import ServiceClient -from bingads.v13.reporting.reporting_service_manager import ReportingServiceManager from suds import sudsobject CACHE: VcrCache = VcrCache() @@ -68,11 +66,10 @@ def additional_fields(self) -> Optional[str]: @property def _service(self) -> Union[ServiceClient, ReportingServiceManager]: return self.client.get_service(service_name=self.service_name) - + @property def _user_id(self) -> int: return self._service.GetUser().User.Id - def next_page_token(self, response: sudsobject.Object, **kwargs: Mapping[str, Any]) -> Optional[Mapping[str, Any]]: """ @@ -101,7 +98,6 @@ def send_request(self, params: Mapping[str, Any], customer_id: str, account_id: else: return request - def read_records( self, sync_mode: SyncMode, @@ -116,9 +112,9 @@ def read_records( while True: params = self.request_params( - stream_state=stream_state, - stream_slice=stream_slice, - next_page_token=next_page_token, + stream_state=stream_state, + stream_slice=stream_slice, + next_page_token=next_page_token, account_id=account_id, ) response = self.send_request(params, customer_id=customer_id, account_id=account_id) @@ -251,7 +247,9 @@ def stream_slices( ) -> Iterable[Optional[Mapping[str, Any]]]: campaigns = Campaigns(self.client, self.config) for account in Accounts(self.client, self.config).read_records(SyncMode.full_refresh): - for campaign in campaigns.read_records(sync_mode=SyncMode.full_refresh, stream_slice={"account_id": account["Id"], "customer_id": account["ParentCustomerId"]}): + for campaign in campaigns.read_records( + sync_mode=SyncMode.full_refresh, stream_slice={"account_id": account["Id"], "customer_id": account["ParentCustomerId"]} + ): yield {"campaign_id": campaign["Id"], "account_id": account["Id"], "customer_id": account["ParentCustomerId"]} yield from [] @@ -581,8 +579,6 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> except Exception as error: return False, error - - def get_report_streams(self, aggregation_type: str) -> List[Stream]: return [ globals()[f"AccountPerformanceReport{aggregation_type}"], From 6b442db08a5784eb5afa0e70d6745987a21f8f96 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Thu, 19 May 2022 12:44:20 +0300 Subject: [PATCH 09/19] fixed broken schema for account stream --- .../source-bing-ads/source_bing_ads/schemas/accounts.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/accounts.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/accounts.json index fc5f671d061e..edce04ef5679 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/accounts.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/accounts.json @@ -80,7 +80,7 @@ "type": ["null", "number"] }, "PauseReason": { - "type": ["null", "string"] + "type": ["null", "number"] }, "PaymentMethodId": { "type": ["null", "number"] From f53386b9231cf3b719672ce12e9ec29d245801d9 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Thu, 19 May 2022 16:34:47 +0300 Subject: [PATCH 10/19] added old config support --- .../acceptance-test-config.yml | 2 ++ .../source-bing-ads/source_bing_ads/client.py | 28 +++++++++++++++---- .../source-bing-ads/source_bing_ads/spec.json | 1 - 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml index 205bc72cbedd..c73cbe55e46c 100644 --- a/airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml @@ -3,6 +3,8 @@ tests: spec: - spec_path: "source_bing_ads/spec.json" connection: + - config_path: "secrets/config_old.json" + status: "succeed" - config_path: "secrets/config.json" status: "succeed" - config_path: "integration_tests/invalid_config.json" diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py index 01ccb57dc184..bbbf6ad79205 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py @@ -36,28 +36,46 @@ class Client: def __init__( self, - credentials: dict, tenant_id: str, reports_start_date: str, hourly_reports: bool, daily_reports: bool, weekly_reports: bool, monthly_reports: bool, + credentials: dict = None, + client_id: str = None, # deprecated + client_secret: str = None, # deprecated + developer_token: str = None, # deprecated + refresh_token: str = None, # deprecated **kwargs: Mapping[str, Any], ) -> None: self.authorization_data: Mapping[str, AuthorizationData] = {} - self.authentication = self._get_auth_client(credentials, tenant_id) - self.refresh_token = credentials["refresh_token"] - self.developer_token = credentials["developer_token"] + self.refresh_token = credentials["refresh_token"] if credentials else refresh_token + self.developer_token = credentials["developer_token"] if credentials else developer_token self.hourly_reports = hourly_reports self.daily_reports = daily_reports self.weekly_reports = weekly_reports self.monthly_reports = monthly_reports - + + self.client_id = client_id # deprecated + self.client_secret = client_secret # deprecated + + self.authentication = self._get_auth_client(credentials, tenant_id) self.oauth: OAuthTokens = self._get_access_token() self.reports_start_date = pendulum.parse(reports_start_date).astimezone(tz=timezone.utc) def _get_auth_client(self, credentials: dict, tenant_id: str) -> OAuthWebAuthCodeGrant: + + # support the deprecated old input configuration + if self.client_id or self.client_secret: + auth_creds = { + "client_id": self.client_id, + "client_secret": self.client_secret, + "redirection_uri": "", # should be empty string + "tenant": tenant_id, + } + return OAuthWebAuthCodeGrant(**auth_creds) + # https://github.com/BingAds/BingAds-Python-SDK/blob/e7b5a618e87a43d0a5e2c79d9aa4626e208797bd/bingads/authorization.py#L390 auth_creds = { "client_id": credentials["client_id"], diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json index 28d2dddac378..571157fcf4ac 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json @@ -5,7 +5,6 @@ "title": "Bing Ads Spec", "type": "object", "required": [ - "credentials", "reports_start_date", "hourly_reports", "daily_reports", From 1cf8f6f35a0eb0347bbcc316416e4ba469224ee6 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Fri, 20 May 2022 15:30:28 +0300 Subject: [PATCH 11/19] added new unit test to increase test coverage to 91% --- .../source-bing-ads/source_bing_ads/client.py | 20 +- ...countperformancereportmonthly_records.json | 332 ++++++++++++++++++ .../unit_tests/accounts_records.json | 82 +++++ .../source-bing-ads/unit_tests/test_source.py | 167 +++++++++ 4 files changed, 591 insertions(+), 10 deletions(-) create mode 100644 airbyte-integrations/connectors/source-bing-ads/unit_tests/accountperformancereportmonthly_records.json create mode 100644 airbyte-integrations/connectors/source-bing-ads/unit_tests/accounts_records.json create mode 100644 airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py index bbbf6ad79205..f483fb0369a1 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py @@ -43,10 +43,10 @@ def __init__( weekly_reports: bool, monthly_reports: bool, credentials: dict = None, - client_id: str = None, # deprecated - client_secret: str = None, # deprecated - developer_token: str = None, # deprecated - refresh_token: str = None, # deprecated + client_id: str = None, # deprecated + client_secret: str = None, # deprecated + developer_token: str = None, # deprecated + refresh_token: str = None, # deprecated **kwargs: Mapping[str, Any], ) -> None: self.authorization_data: Mapping[str, AuthorizationData] = {} @@ -56,16 +56,16 @@ def __init__( self.daily_reports = daily_reports self.weekly_reports = weekly_reports self.monthly_reports = monthly_reports - - self.client_id = client_id # deprecated - self.client_secret = client_secret # deprecated - + + self.client_id = client_id # deprecated + self.client_secret = client_secret # deprecated + self.authentication = self._get_auth_client(credentials, tenant_id) self.oauth: OAuthTokens = self._get_access_token() self.reports_start_date = pendulum.parse(reports_start_date).astimezone(tz=timezone.utc) def _get_auth_client(self, credentials: dict, tenant_id: str) -> OAuthWebAuthCodeGrant: - + # support the deprecated old input configuration if self.client_id or self.client_secret: auth_creds = { @@ -75,7 +75,7 @@ def _get_auth_client(self, credentials: dict, tenant_id: str) -> OAuthWebAuthCod "tenant": tenant_id, } return OAuthWebAuthCodeGrant(**auth_creds) - + # https://github.com/BingAds/BingAds-Python-SDK/blob/e7b5a618e87a43d0a5e2c79d9aa4626e208797bd/bingads/authorization.py#L390 auth_creds = { "client_id": credentials["client_id"], diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/accountperformancereportmonthly_records.json b/airbyte-integrations/connectors/source-bing-ads/unit_tests/accountperformancereportmonthly_records.json new file mode 100644 index 000000000000..09e07640539c --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/accountperformancereportmonthly_records.json @@ -0,0 +1,332 @@ +[{ + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-06-01", + "DeviceType": "Computer", + "Network": "Bing and Yahoo! search", + "Impressions": 1380, + "Clicks": 22, + "Spend": 3.34, + "Ctr": 0.0159, + "AverageCpc": 0.15, + "ReturnOnAdSpend": 0.0, + "RevenuePerConversion": null, + "ConversionRate": null + }, { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-06-01", + "DeviceType": "Computer", + "Network": "Syndicated search partners", + "Impressions": 1992, + "Clicks": 25, + "Spend": 4.62, + "Ctr": 0.0126, + "AverageCpc": 0.18, + "ReturnOnAdSpend": 0.0, + "RevenuePerConversion": null, + "ConversionRate": null + }, { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-06-01", + "DeviceType": "Computer", + "Network": "AOL search", + "Impressions": 2, + "Clicks": 0, + "Spend": 0.0, + "Ctr": 0.0, + "AverageCpc": 0.0, + "ReturnOnAdSpend": null, + "RevenuePerConversion": null, + "ConversionRate": null + }, { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-06-01", + "DeviceType": "Computer", + "Network": "Audience", + "Impressions": 9, + "Clicks": 0, + "Spend": 0.0, + "Ctr": 0.0, + "AverageCpc": 0.0, + "ReturnOnAdSpend": null, + "RevenuePerConversion": null, + "ConversionRate": null + }, { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-06-01", + "DeviceType": "Smartphone", + "Network": "Bing and Yahoo! search", + "Impressions": 73, + "Clicks": 5, + "Spend": 0.98, + "Ctr": 0.06849999999999999, + "AverageCpc": 0.2, + "ReturnOnAdSpend": 0.0, + "RevenuePerConversion": null, + "ConversionRate": null + }, { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-06-01", + "DeviceType": "Smartphone", + "Network": "Syndicated search partners", + "Impressions": 5754, + "Clicks": 38, + "Spend": 7.1, + "Ctr": 0.0066, + "AverageCpc": 0.19, + "ReturnOnAdSpend": 0.0, + "RevenuePerConversion": null, + "ConversionRate": null + }, { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-06-01", + "DeviceType": "Tablet", + "Network": "Bing and Yahoo! search", + "Impressions": 1, + "Clicks": 0, + "Spend": 0.0, + "Ctr": 0.0, + "AverageCpc": 0.0, + "ReturnOnAdSpend": null, + "RevenuePerConversion": null, + "ConversionRate": null + }, { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-06-01", + "DeviceType": "Tablet", + "Network": "Syndicated search partners", + "Impressions": 82, + "Clicks": 1, + "Spend": 0.22, + "Ctr": 0.012199999999999999, + "AverageCpc": 0.22, + "ReturnOnAdSpend": 0.0, + "RevenuePerConversion": null, + "ConversionRate": null + }, { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-06-01", + "DeviceType": "Tablet", + "Network": "AOL search", + "Impressions": 0, + "Clicks": 0, + "Spend": 0.0, + "Ctr": null, + "AverageCpc": 0.0, + "ReturnOnAdSpend": null, + "RevenuePerConversion": null, + "ConversionRate": null + }, { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-07-01", + "DeviceType": "Computer", + "Network": "Bing and Yahoo! search", + "Impressions": 783, + "Clicks": 12, + "Spend": 1.23, + "Ctr": 0.015300000000000001, + "AverageCpc": 0.1, + "ReturnOnAdSpend": 0.0, + "RevenuePerConversion": null, + "ConversionRate": null + }, { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-07-01", + "DeviceType": "Computer", + "Network": "Syndicated search partners", + "Impressions": 1712, + "Clicks": 23, + "Spend": 2.72, + "Ctr": 0.0134, + "AverageCpc": 0.12, + "ReturnOnAdSpend": 0.0, + "RevenuePerConversion": null, + "ConversionRate": null + }, { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-07-01", + "DeviceType": "Computer", + "Network": "AOL search", + "Impressions": 0, + "Clicks": 0, + "Spend": 0.0, + "Ctr": null, + "AverageCpc": 0.0, + "ReturnOnAdSpend": null, + "RevenuePerConversion": null, + "ConversionRate": null + }, { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-07-01", + "DeviceType": "Computer", + "Network": "Audience", + "Impressions": 9, + "Clicks": 0, + "Spend": 0.0, + "Ctr": 0.0, + "AverageCpc": 0.0, + "ReturnOnAdSpend": null, + "RevenuePerConversion": null, + "ConversionRate": null + }, { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-07-01", + "DeviceType": "Smartphone", + "Network": "Bing and Yahoo! search", + "Impressions": 52, + "Clicks": 3, + "Spend": 0.19, + "Ctr": 0.057699999999999994, + "AverageCpc": 0.06, + "ReturnOnAdSpend": 0.0, + "RevenuePerConversion": null, + "ConversionRate": null + }, { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-07-01", + "DeviceType": "Smartphone", + "Network": "Syndicated search partners", + "Impressions": 38546, + "Clicks": 201, + "Spend": 18.1, + "Ctr": 0.0052, + "AverageCpc": 0.09, + "ReturnOnAdSpend": 0.0, + "RevenuePerConversion": null, + "ConversionRate": null + }, { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-07-01", + "DeviceType": "Tablet", + "Network": "Bing and Yahoo! search", + "Impressions": 1, + "Clicks": 0, + "Spend": 0.0, + "Ctr": 0.0, + "AverageCpc": 0.0, + "ReturnOnAdSpend": null, + "RevenuePerConversion": null, + "ConversionRate": null + }, { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-07-01", + "DeviceType": "Tablet", + "Network": "Syndicated search partners", + "Impressions": 729, + "Clicks": 7, + "Spend": 0.55, + "Ctr": 0.0096, + "AverageCpc": 0.08, + "ReturnOnAdSpend": 0.0, + "RevenuePerConversion": null, + "ConversionRate": null + }, { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-08-01", + "DeviceType": "Computer", + "Network": "Bing and Yahoo! search", + "Impressions": 22, + "Clicks": 0, + "Spend": 0.0, + "Ctr": 0.0, + "AverageCpc": 0.0, + "ReturnOnAdSpend": null, + "RevenuePerConversion": null, + "ConversionRate": null + }, { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-08-01", + "DeviceType": "Computer", + "Network": "Syndicated search partners", + "Impressions": 60, + "Clicks": 1, + "Spend": 0.11, + "Ctr": 0.0167, + "AverageCpc": 0.11, + "ReturnOnAdSpend": 0.0, + "RevenuePerConversion": null, + "ConversionRate": null + }, { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-08-01", + "DeviceType": "Smartphone", + "Network": "Bing and Yahoo! search", + "Impressions": 1, + "Clicks": 0, + "Spend": 0.0, + "Ctr": 0.0, + "AverageCpc": 0.0, + "ReturnOnAdSpend": null, + "RevenuePerConversion": null, + "ConversionRate": null + }, { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-08-01", + "DeviceType": "Smartphone", + "Network": "Syndicated search partners", + "Impressions": 1438, + "Clicks": 22, + "Spend": 1.62, + "Ctr": 0.015300000000000001, + "AverageCpc": 0.07, + "ReturnOnAdSpend": 0.0, + "RevenuePerConversion": null, + "ConversionRate": null + }, { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-08-01", + "DeviceType": "Tablet", + "Network": "Syndicated search partners", + "Impressions": 7, + "Clicks": 0, + "Spend": 0.0, + "Ctr": 0.0, + "AverageCpc": 0.0, + "ReturnOnAdSpend": null, + "RevenuePerConversion": null, + "ConversionRate": null + } +] diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/accounts_records.json b/airbyte-integrations/connectors/source-bing-ads/unit_tests/accounts_records.json new file mode 100644 index 000000000000..c1f19d917c65 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/accounts_records.json @@ -0,0 +1,82 @@ +[{ + "BillToCustomerId": 251186883, + "CurrencyCode": "USD", + "AccountFinancialStatus": "ClearFinancialStatus", + "Id": 180519267, + "Language": "English", + "LastModifiedByUserId": 138225488, + "LastModifiedTime": "2021-07-09T13:16:43.337000", + "Name": "Airbyte", + "Number": "F149MJ18", + "ParentCustomerId": 251186883, + "PaymentMethodId": null, + "PaymentMethodType": null, + "PrimaryUserId": 138225488, + "AccountLifeCycleStatus": "Pending", + "TimeStamp": "AAAAAEpme9E=", + "TimeZone": "CentralTimeUSCanada", + "PauseReason": null, + "ForwardCompatibilityMap": null, + "LinkedAgencies": null, + "SalesHouseCustomerId": null, + "TaxInformation": null, + "BackUpPaymentInstrumentId": null, + "BillingThresholdAmount": null, + "BusinessAddress": { + "City": "San Francisco", + "CountryCode": "US", + "Id": 149649761, + "Line1": "350 29th Ave", + "Line2": null, + "Line3": null, + "Line4": null, + "PostalCode": "94121-1703", + "StateOrProvince": "CA", + "TimeStamp": null, + "BusinessName": "Airbyte" + }, + "AutoTagType": "Preserve", + "SoldToPaymentInstrumentId": null, + "AccountMode": "Expert" + }, { + "BillToCustomerId": 251186883, + "CurrencyCode": "USD", + "AccountFinancialStatus": "ClearFinancialStatus", + "Id": 180278106, + "Language": "English", + "LastModifiedByUserId": 3, + "LastModifiedTime": "2021-08-23T07:06:19.147000", + "Name": "Daxtarity Inc.", + "Number": "F149GKV5", + "ParentCustomerId": 251186883, + "PaymentMethodId": 138188746, + "PaymentMethodType": "CreditCard", + "PrimaryUserId": 138225488, + "AccountLifeCycleStatus": "Active", + "TimeStamp": "AAAAAE0a41E=", + "TimeZone": "Arizona", + "PauseReason": null, + "ForwardCompatibilityMap": null, + "LinkedAgencies": null, + "SalesHouseCustomerId": null, + "TaxInformation": null, + "BackUpPaymentInstrumentId": null, + "BillingThresholdAmount": null, + "BusinessAddress": { + "City": "San Francisco", + "CountryCode": "US", + "Id": 149004358, + "Line1": "350 29th avenue", + "Line2": null, + "Line3": null, + "Line4": null, + "PostalCode": "94121", + "StateOrProvince": "CA", + "TimeStamp": null, + "BusinessName": "Daxtarity Inc." + }, + "AutoTagType": "Inactive", + "SoldToPaymentInstrumentId": null, + "AccountMode": "Expert" + } +] \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py new file mode 100644 index 000000000000..90ea4420492a --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py @@ -0,0 +1,167 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +import json +from unittest.mock import MagicMock, patch + +import pytest +import source_bing_ads +from airbyte_cdk.models import SyncMode +from source_bing_ads.client import Client +from source_bing_ads.source import AccountPerformanceReportMonthly, Accounts, AdGroups, Ads, Campaigns, SourceBingAds + + +@pytest.fixture(name="config") +def config_fixture(): + """Generates streams settings from a config file""" + CONFIG_FILE = "secrets/config.json" + with open(CONFIG_FILE, "r") as f: + return json.loads(f.read()) + + +@pytest.fixture(name="logger_mock") +def logger_mock_fixture(): + return patch("source_bing_ads.source.AirbyteLogger") + + +@patch.object(source_bing_ads.source, "Client") +def test_streams_config_based(mocked_client, config): + streams = SourceBingAds().streams(config) + assert len(streams) == 15 + + +@patch.object(source_bing_ads.source, "Client") +def test_streams_all(mocked_client): + streams = SourceBingAds().streams(MagicMock()) + assert len(streams) == 25 + + +@patch.object(source_bing_ads.source, "Client") +def test_source_check_connection_ok(mocked_client, config, logger_mock): + with patch.object(Accounts, "read_records", return_value=iter([{"Id": 180519267}, {"Id": 180278106}])): + assert SourceBingAds().check_connection(logger_mock, config=config) == (True, None) + + +@patch.object(source_bing_ads.source, "Client") +def test_source_check_connection_ok_strategy_all(mocked_client, config, logger_mock): + config = config.copy() + config["accounts"]["selection_strategy"] = "all" + with patch.object(Accounts, "read_records", return_value=iter([{"Id": 180519267}, {"Id": 180278106}])): + assert SourceBingAds().check_connection(logger_mock, config=config) == (True, None) + + +@patch.object(source_bing_ads.source, "Client") +def test_source_check_connection_failed(mocked_client, config, logger_mock): + with patch.object(Accounts, "read_records", return_value=0): + assert SourceBingAds().check_connection(logger_mock, config=config)[0] is False + + +@patch.object(source_bing_ads.source, "Client") +def test_campaigns_request_params(mocked_client, config): + + campaigns = Campaigns(mocked_client, config) + + request_params = campaigns.request_params(stream_slice={"account_id": "account_id"}) + assert request_params == { + "AccountId": "account_id", + "ReturnAdditionalFields": "AdScheduleUseSearcherTimeZone BidStrategyId CpvCpmBiddingScheme DynamicDescriptionSetting DynamicFeedSetting MaxConversionValueBiddingScheme MultimediaAdsBidAdjustment TargetImpressionShareBiddingScheme TargetSetting VerifiedTrackingSetting", + } + + +@patch.object(source_bing_ads.source, "Client") +def test_campaigns_stream_slices(mocked_client, config): + + campaigns = Campaigns(mocked_client, config) + accounts_read_records = iter([{"Id": 180519267, "ParentCustomerId": 100}, {"Id": 180278106, "ParentCustomerId": 200}]) + with patch.object(Accounts, "read_records", return_value=accounts_read_records): + slices = campaigns.stream_slices() + assert list(slices) == [ + {"account_id": 180519267, "customer_id": 100}, + {"account_id": 180278106, "customer_id": 200}, + ] + + +@patch.object(source_bing_ads.source, "Client") +def test_adgroups_stream_slices(mocked_client, config): + + adgroups = AdGroups(mocked_client, config) + accounts_read_records = iter([{"Id": 180519267, "ParentCustomerId": 100}, {"Id": 180278106, "ParentCustomerId": 200}]) + campaigns_read_records = [iter([{"Id": 11}, {"Id": 22}]), iter([{"Id": 55}, {"Id": 66}])] + with patch.object(Accounts, "read_records", return_value=accounts_read_records): + with patch.object(Campaigns, "read_records", side_effect=campaigns_read_records): + slices = adgroups.stream_slices() + assert list(slices) == [ + {"campaign_id": 11, "account_id": 180519267, "customer_id": 100}, + {"campaign_id": 22, "account_id": 180519267, "customer_id": 100}, + {"campaign_id": 55, "account_id": 180278106, "customer_id": 200}, + {"campaign_id": 66, "account_id": 180278106, "customer_id": 200}, + ] + + +@patch.object(source_bing_ads.source, "Client") +def test_ads_request_params(mocked_client, config): + + ads = Ads(mocked_client, config) + + request_params = ads.request_params(stream_slice={"ad_group_id": "ad_group_id"}) + assert request_params == { + "AdGroupId": "ad_group_id", + "AdTypes": { + "AdType": ["Text", "Image", "Product", "AppInstall", "ExpandedText", "DynamicSearch", "ResponsiveAd", "ResponsiveSearch"] + }, + "ReturnAdditionalFields": "ImpressionTrackingUrls Videos LongHeadlines", + } + + +@patch.object(source_bing_ads.source, "Client") +def test_ads_stream_slices(mocked_client, config): + + ads = Ads(mocked_client, config) + + with patch.object( + AdGroups, + "stream_slices", + return_value=iter([{"account_id": 180519267, "customer_id": 100}, {"account_id": 180278106, "customer_id": 200}]), + ): + with patch.object(AdGroups, "read_records", side_effect=[iter([{"Id": 11}, {"Id": 22}]), iter([{"Id": 55}, {"Id": 66}])]): + slices = ads.stream_slices() + assert list(slices) == [ + {"ad_group_id": 11, "account_id": 180519267, "customer_id": 100}, + {"ad_group_id": 22, "account_id": 180519267, "customer_id": 100}, + {"ad_group_id": 55, "account_id": 180278106, "customer_id": 200}, + {"ad_group_id": 66, "account_id": 180278106, "customer_id": 200}, + ] + + +@patch.object(source_bing_ads.source, "Client") +def test_AccountPerformanceReportMonthly_request_params(mocked_client, config): + + accountperformancereportmonthly = AccountPerformanceReportMonthly(mocked_client, config) + request_params = accountperformancereportmonthly.request_params(account_id=180278106) + del request_params["report_request"] + assert request_params == { + "overwrite_result_file": True, + # 'report_request': , + "result_file_directory": "/tmp", + "result_file_name": "AccountPerformanceReport", + "timeout_in_milliseconds": 300000, + } + + +def test_accounts_live(config): + client = Client(**config) + accounts = Accounts(client, config) + records = accounts.read_records(SyncMode.full_refresh) + assert len(list(records)) == 4 + + +# def test_AccountPerformanceReportMonthly_live(config): +# +# with open("./unit_tests/accountperformancereportmonthly_records.json", "r") as f: +# accountperformancereportmonthly_records = json.load(f) +# +# client = Client(**config) +# accountperformancereportmonthly = AccountPerformanceReportMonthly(client, config) +# records = accountperformancereportmonthly.read_records(SyncMode.full_refresh, stream_slice={"account_id": 180278106}) +# assert list(records) == accountperformancereportmonthly_records From a0b18133a84b94208a07bf872e31f98a82fadf16 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Fri, 20 May 2022 16:59:15 +0300 Subject: [PATCH 12/19] removed unneccessary tests --- .../connectors/source-bing-ads/unit_tests/test_source.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py index 90ea4420492a..e78d0385cfa4 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py @@ -43,14 +43,6 @@ def test_source_check_connection_ok(mocked_client, config, logger_mock): assert SourceBingAds().check_connection(logger_mock, config=config) == (True, None) -@patch.object(source_bing_ads.source, "Client") -def test_source_check_connection_ok_strategy_all(mocked_client, config, logger_mock): - config = config.copy() - config["accounts"]["selection_strategy"] = "all" - with patch.object(Accounts, "read_records", return_value=iter([{"Id": 180519267}, {"Id": 180278106}])): - assert SourceBingAds().check_connection(logger_mock, config=config) == (True, None) - - @patch.object(source_bing_ads.source, "Client") def test_source_check_connection_failed(mocked_client, config, logger_mock): with patch.object(Accounts, "read_records", return_value=0): From 4766c1e683bed55629df1da96a655c4086c2cc61 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Fri, 20 May 2022 17:28:29 +0300 Subject: [PATCH 13/19] formated code --- .../integration_tests/invalid_config.json | 28 ++++---- .../source-bing-ads/source_bing_ads/spec.json | 8 ++- ...countperformancereportmonthly_records.json | 66 ++++++++++++------- .../unit_tests/accounts_records.json | 8 ++- 4 files changed, 70 insertions(+), 40 deletions(-) diff --git a/airbyte-integrations/connectors/source-bing-ads/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-bing-ads/integration_tests/invalid_config.json index 6cf658fbc36c..691be4f6e091 100644 --- a/airbyte-integrations/connectors/source-bing-ads/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-bing-ads/integration_tests/invalid_config.json @@ -1,15 +1,15 @@ -{ - "credentials": { - "auth_method": "oauth2.0", - "developer_token": "asgag4gwag3", - "refresh_token": "as2Ggas23gsa236gasgaskjfhas7i8ygf78as7osa7gy87asg8as7tg6as", - "client_secret": "1234", - "client_id": "123" - }, - "reports_start_date": "2018-11-13", - "hourly_reports": false, - "daily_reports": false, - "weekly_reports": true, - "monthly_reports": true, - "tenant_id": "common" +{ + "credentials": { + "auth_method": "oauth2.0", + "developer_token": "asgag4gwag3", + "refresh_token": "as2Ggas23gsa236gasgaskjfhas7i8ygf78as7osa7gy87asg8as7tg6as", + "client_secret": "1234", + "client_id": "123" + }, + "reports_start_date": "2018-11-13", + "hourly_reports": false, + "daily_reports": false, + "weekly_reports": true, + "monthly_reports": true, + "tenant_id": "common" } diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json index 571157fcf4ac..5cd0bf1212b8 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json @@ -72,7 +72,13 @@ "type": "object", "title": "Private OAuth2.0 Client", "description": "Microsoft Private Client API Credentials for connecting to Bings Ads", - "required": ["auth_method", "developer_token", "client_id", "client_secret", "refresh_token"], + "required": [ + "auth_method", + "developer_token", + "client_id", + "client_secret", + "refresh_token" + ], "properties": { "auth_method": { "type": "string", diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/accountperformancereportmonthly_records.json b/airbyte-integrations/connectors/source-bing-ads/unit_tests/accountperformancereportmonthly_records.json index 09e07640539c..9f8ee44fbb6a 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/accountperformancereportmonthly_records.json +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/accountperformancereportmonthly_records.json @@ -1,4 +1,5 @@ -[{ +[ + { "AccountName": "Daxtarity Inc.", "AccountNumber": "F149GKV5", "AccountId": 180278106, @@ -13,7 +14,8 @@ "ReturnOnAdSpend": 0.0, "RevenuePerConversion": null, "ConversionRate": null - }, { + }, + { "AccountName": "Daxtarity Inc.", "AccountNumber": "F149GKV5", "AccountId": 180278106, @@ -28,7 +30,8 @@ "ReturnOnAdSpend": 0.0, "RevenuePerConversion": null, "ConversionRate": null - }, { + }, + { "AccountName": "Daxtarity Inc.", "AccountNumber": "F149GKV5", "AccountId": 180278106, @@ -43,7 +46,8 @@ "ReturnOnAdSpend": null, "RevenuePerConversion": null, "ConversionRate": null - }, { + }, + { "AccountName": "Daxtarity Inc.", "AccountNumber": "F149GKV5", "AccountId": 180278106, @@ -58,7 +62,8 @@ "ReturnOnAdSpend": null, "RevenuePerConversion": null, "ConversionRate": null - }, { + }, + { "AccountName": "Daxtarity Inc.", "AccountNumber": "F149GKV5", "AccountId": 180278106, @@ -73,7 +78,8 @@ "ReturnOnAdSpend": 0.0, "RevenuePerConversion": null, "ConversionRate": null - }, { + }, + { "AccountName": "Daxtarity Inc.", "AccountNumber": "F149GKV5", "AccountId": 180278106, @@ -88,7 +94,8 @@ "ReturnOnAdSpend": 0.0, "RevenuePerConversion": null, "ConversionRate": null - }, { + }, + { "AccountName": "Daxtarity Inc.", "AccountNumber": "F149GKV5", "AccountId": 180278106, @@ -103,7 +110,8 @@ "ReturnOnAdSpend": null, "RevenuePerConversion": null, "ConversionRate": null - }, { + }, + { "AccountName": "Daxtarity Inc.", "AccountNumber": "F149GKV5", "AccountId": 180278106, @@ -118,7 +126,8 @@ "ReturnOnAdSpend": 0.0, "RevenuePerConversion": null, "ConversionRate": null - }, { + }, + { "AccountName": "Daxtarity Inc.", "AccountNumber": "F149GKV5", "AccountId": 180278106, @@ -133,7 +142,8 @@ "ReturnOnAdSpend": null, "RevenuePerConversion": null, "ConversionRate": null - }, { + }, + { "AccountName": "Daxtarity Inc.", "AccountNumber": "F149GKV5", "AccountId": 180278106, @@ -148,7 +158,8 @@ "ReturnOnAdSpend": 0.0, "RevenuePerConversion": null, "ConversionRate": null - }, { + }, + { "AccountName": "Daxtarity Inc.", "AccountNumber": "F149GKV5", "AccountId": 180278106, @@ -163,7 +174,8 @@ "ReturnOnAdSpend": 0.0, "RevenuePerConversion": null, "ConversionRate": null - }, { + }, + { "AccountName": "Daxtarity Inc.", "AccountNumber": "F149GKV5", "AccountId": 180278106, @@ -178,7 +190,8 @@ "ReturnOnAdSpend": null, "RevenuePerConversion": null, "ConversionRate": null - }, { + }, + { "AccountName": "Daxtarity Inc.", "AccountNumber": "F149GKV5", "AccountId": 180278106, @@ -193,7 +206,8 @@ "ReturnOnAdSpend": null, "RevenuePerConversion": null, "ConversionRate": null - }, { + }, + { "AccountName": "Daxtarity Inc.", "AccountNumber": "F149GKV5", "AccountId": 180278106, @@ -208,7 +222,8 @@ "ReturnOnAdSpend": 0.0, "RevenuePerConversion": null, "ConversionRate": null - }, { + }, + { "AccountName": "Daxtarity Inc.", "AccountNumber": "F149GKV5", "AccountId": 180278106, @@ -223,7 +238,8 @@ "ReturnOnAdSpend": 0.0, "RevenuePerConversion": null, "ConversionRate": null - }, { + }, + { "AccountName": "Daxtarity Inc.", "AccountNumber": "F149GKV5", "AccountId": 180278106, @@ -238,7 +254,8 @@ "ReturnOnAdSpend": null, "RevenuePerConversion": null, "ConversionRate": null - }, { + }, + { "AccountName": "Daxtarity Inc.", "AccountNumber": "F149GKV5", "AccountId": 180278106, @@ -253,7 +270,8 @@ "ReturnOnAdSpend": 0.0, "RevenuePerConversion": null, "ConversionRate": null - }, { + }, + { "AccountName": "Daxtarity Inc.", "AccountNumber": "F149GKV5", "AccountId": 180278106, @@ -268,7 +286,8 @@ "ReturnOnAdSpend": null, "RevenuePerConversion": null, "ConversionRate": null - }, { + }, + { "AccountName": "Daxtarity Inc.", "AccountNumber": "F149GKV5", "AccountId": 180278106, @@ -283,7 +302,8 @@ "ReturnOnAdSpend": 0.0, "RevenuePerConversion": null, "ConversionRate": null - }, { + }, + { "AccountName": "Daxtarity Inc.", "AccountNumber": "F149GKV5", "AccountId": 180278106, @@ -298,7 +318,8 @@ "ReturnOnAdSpend": null, "RevenuePerConversion": null, "ConversionRate": null - }, { + }, + { "AccountName": "Daxtarity Inc.", "AccountNumber": "F149GKV5", "AccountId": 180278106, @@ -313,7 +334,8 @@ "ReturnOnAdSpend": 0.0, "RevenuePerConversion": null, "ConversionRate": null - }, { + }, + { "AccountName": "Daxtarity Inc.", "AccountNumber": "F149GKV5", "AccountId": 180278106, diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/accounts_records.json b/airbyte-integrations/connectors/source-bing-ads/unit_tests/accounts_records.json index c1f19d917c65..44728c41134c 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/accounts_records.json +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/accounts_records.json @@ -1,4 +1,5 @@ -[{ +[ + { "BillToCustomerId": 251186883, "CurrencyCode": "USD", "AccountFinancialStatus": "ClearFinancialStatus", @@ -38,7 +39,8 @@ "AutoTagType": "Preserve", "SoldToPaymentInstrumentId": null, "AccountMode": "Expert" - }, { + }, + { "BillToCustomerId": 251186883, "CurrencyCode": "USD", "AccountFinancialStatus": "ClearFinancialStatus", @@ -79,4 +81,4 @@ "SoldToPaymentInstrumentId": null, "AccountMode": "Expert" } -] \ No newline at end of file +] From b5ac71c335156729a5682a8ab7ea53b800323eb1 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Sat, 21 May 2022 22:16:40 +0300 Subject: [PATCH 14/19] updated after review --- .../source-bing-ads/unit_tests/test_source.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py index e78d0385cfa4..9c5a4376f19e 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py @@ -146,14 +146,3 @@ def test_accounts_live(config): accounts = Accounts(client, config) records = accounts.read_records(SyncMode.full_refresh) assert len(list(records)) == 4 - - -# def test_AccountPerformanceReportMonthly_live(config): -# -# with open("./unit_tests/accountperformancereportmonthly_records.json", "r") as f: -# accountperformancereportmonthly_records = json.load(f) -# -# client = Client(**config) -# accountperformancereportmonthly = AccountPerformanceReportMonthly(client, config) -# records = accountperformancereportmonthly.read_records(SyncMode.full_refresh, stream_slice={"account_id": 180278106}) -# assert list(records) == accountperformancereportmonthly_records From 4836cd4a2da54a5ecdf81c0c7eea2d504621a19a Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Sun, 22 May 2022 00:49:07 +0300 Subject: [PATCH 15/19] added oauth implementation java test --- .../flows/MicrosoftBingAdsOAuthFlow.java | 13 ++++- .../flows/MicrosoftBingAdsOAuthFlowTest.java | 47 +++++++++++++++++++ 2 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 airbyte-oauth/src/test/java/io/airbyte/oauth/flows/MicrosoftBingAdsOAuthFlowTest.java diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/MicrosoftBingAdsOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/MicrosoftBingAdsOAuthFlow.java index 0a5c0688274f..b355d14b92be 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/MicrosoftBingAdsOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/MicrosoftBingAdsOAuthFlow.java @@ -5,6 +5,7 @@ package io.airbyte.oauth.flows; import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.oauth.BaseOAuth2Flow; @@ -18,10 +19,13 @@ public class MicrosoftBingAdsOAuthFlow extends BaseOAuth2Flow { + private static final String fieldName = "tenant_id"; + public MicrosoftBingAdsOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient) { super(configRepository, httpClient); } + @VisibleForTesting public MicrosoftBingAdsOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient, final Supplier stateSupplier) { super(configRepository, httpClient, stateSupplier); } @@ -37,7 +41,12 @@ protected String formatConsentUrl(final UUID definitionId, final JsonNode inputOAuthConfiguration) throws IOException { - final String tenantId = getConfigValueUnsafe(inputOAuthConfiguration, "tenant_id"); + final String tenantId; + try { + tenantId = getConfigValueUnsafe(inputOAuthConfiguration, fieldName); + } catch (final IllegalArgumentException e) { + throw new IOException("Failed to get " + fieldName + " value from input configuration", e); + } try { return new URIBuilder() @@ -70,7 +79,7 @@ protected Map getAccessTokenQueryParameters(final String clientI @Override protected String getAccessTokenUrl(final JsonNode inputOAuthConfiguration) { - final String tenantId = getConfigValueUnsafe(inputOAuthConfiguration, "tenant_id"); + final String tenantId = getConfigValueUnsafe(inputOAuthConfiguration, fieldName); return "https://login.microsoftonline.com/" + tenantId + "/oauth2/v2.0/token"; } diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/MicrosoftBingAdsOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/MicrosoftBingAdsOAuthFlowTest.java new file mode 100644 index 000000000000..7a81e9790fec --- /dev/null +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/MicrosoftBingAdsOAuthFlowTest.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.oauth.flows; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.oauth.BaseOAuthFlow; +import java.util.Map; +import org.junit.jupiter.api.Test; + +public class MicrosoftBingAdsOAuthFlowTest extends BaseOAuthFlowTest { + + @Override + protected BaseOAuthFlow getOAuthFlow() { + return new MicrosoftBingAdsOAuthFlow(getConfigRepository(), getHttpClient(), this::getConstantState); + } + + @Override + protected String getExpectedConsentUrl() { + return "https://login.microsoftonline.com/test_tenant_id/oauth2/v2.0/authorize?client_id=test_client_id&response_type=code&redirect_uri=https%3A%2F%2Fairbyte.io&response_mode=query&state=state&scope=offline_access%20https://ads.microsoft.com/msads.manage"; + } + + @Override + protected JsonNode getInputOAuthConfiguration() { + return Jsons.jsonNode(Map.of("tenant_id", "test_tenant_id")); + } + + @Override + protected JsonNode getUserInputFromConnectorConfigSpecification() { + return getJsonSchema(Map.of("tenant_id", Map.of("type", "string"))); + } + + @Test + public void testEmptyInputCompleteDestinationOAuth() {} + + @Test + public void testDeprecatedCompleteDestinationOAuth() {} + + @Test + public void testDeprecatedCompleteSourceOAuth() {} + + @Test + public void testEmptyInputCompleteSourceOAuth() {} + +} From 307bb2064f33081bbcd524ca655515a2829375cb Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Mon, 23 May 2022 16:43:22 +0300 Subject: [PATCH 16/19] updated spec --- .../integration_tests/invalid_config.json | 2 +- .../source-bing-ads/source_bing_ads/client.py | 30 ++++----- .../source-bing-ads/source_bing_ads/spec.json | 61 ++++++------------- 3 files changed, 36 insertions(+), 57 deletions(-) diff --git a/airbyte-integrations/connectors/source-bing-ads/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-bing-ads/integration_tests/invalid_config.json index 691be4f6e091..74f21b9ab9e6 100644 --- a/airbyte-integrations/connectors/source-bing-ads/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-bing-ads/integration_tests/invalid_config.json @@ -1,11 +1,11 @@ { "credentials": { "auth_method": "oauth2.0", - "developer_token": "asgag4gwag3", "refresh_token": "as2Ggas23gsa236gasgaskjfhas7i8ygf78as7osa7gy87asg8as7tg6as", "client_secret": "1234", "client_id": "123" }, + "developer_token": "asgag4gwag3", "reports_start_date": "2018-11-13", "hourly_reports": false, "daily_reports": false, diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py index f483fb0369a1..1f9242f94829 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py @@ -43,15 +43,15 @@ def __init__( weekly_reports: bool, monthly_reports: bool, credentials: dict = None, + developer_token: str = None, client_id: str = None, # deprecated client_secret: str = None, # deprecated - developer_token: str = None, # deprecated refresh_token: str = None, # deprecated **kwargs: Mapping[str, Any], ) -> None: self.authorization_data: Mapping[str, AuthorizationData] = {} self.refresh_token = credentials["refresh_token"] if credentials else refresh_token - self.developer_token = credentials["developer_token"] if credentials else developer_token + self.developer_token = developer_token self.hourly_reports = hourly_reports self.daily_reports = daily_reports self.weekly_reports = weekly_reports @@ -65,24 +65,24 @@ def __init__( self.reports_start_date = pendulum.parse(reports_start_date).astimezone(tz=timezone.utc) def _get_auth_client(self, credentials: dict, tenant_id: str) -> OAuthWebAuthCodeGrant: - - # support the deprecated old input configuration - if self.client_id or self.client_secret: - auth_creds = { - "client_id": self.client_id, - "client_secret": self.client_secret, - "redirection_uri": "", # should be empty string - "tenant": tenant_id, - } - return OAuthWebAuthCodeGrant(**auth_creds) - + + # base auth params # https://github.com/BingAds/BingAds-Python-SDK/blob/e7b5a618e87a43d0a5e2c79d9aa4626e208797bd/bingads/authorization.py#L390 auth_creds = { - "client_id": credentials["client_id"], + "client_id": None, "client_secret": None, "redirection_uri": "", # should be empty string "tenant": tenant_id, - } + } + + # support the deprecated old input configuration + if self.client_id or self.client_secret: + auth_creds["client_id"] = self.client_id + auth_creds["client_secret"] = self.client_secret + return OAuthWebAuthCodeGrant(**auth_creds) + + auth_creds["client_id"] = credentials["client_id"] + if credentials["auth_method"] == "private_client": # the `client_secret` should be provided for `non-public clients` ONLY # https://docs.microsoft.com/en-us/advertising/guides/authentication-oauth-get-tokens?view=bingads-13#request-accesstoken diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json index 5cd0bf1212b8..00e585df1ce4 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json @@ -5,6 +5,7 @@ "title": "Bing Ads Spec", "type": "object", "required": [ + "developer_token", "reports_start_date", "hourly_reports", "daily_reports", @@ -24,57 +25,42 @@ "credentials": { "type": "object", "title": "Authentication Method", - "description": "The authentication method to use to retrieve data from Microsoft Bings Ads", + "description": "The authentication method to use to retrieve data from Microsoft Bing Ads", "order": 1, "oneOf": [ { "type": "object", - "title": "Microsoft OAuth2.0", - "description": "Microsoft API Credentials for connecting to Bings Ads", - "required": ["auth_method", "developer_token"], + "title": "OAuth2.0", + "description": "Microsoft API Credentials for connecting to Bing Ads", + "required": ["auth_method"], "properties": { "auth_method": { "type": "string", "const": "oauth2.0", "order": 0 }, - "developer_token": { - "type": "string", - "title": "Developer Token", - "description": "Developer token associated with user. See more info in the docs.", - "airbyte_secret": true, - "order": 1 - }, "client_id": { "type": "string", "title": "Client ID", "description": "The Client ID of your Microsoft Advertising developer application.", "airbyte_secret": true, - "order": 2 - }, - "client_secret": { - "type": "string", - "title": "Client Secret", - "description": "The Client Secret of your Microsoft Advertising developer application.", - "airbyte_secret": true, - "order": 3 + "order": 1 }, "refresh_token": { "type": "string", "title": "Refresh Token", "description": "Refresh Token to renew the expired Access Token.", "airbyte_secret": true, - "order": 4 + "order": 2 } } }, { "type": "object", - "title": "Private OAuth2.0 Client", - "description": "Microsoft Private Client API Credentials for connecting to Bings Ads", + "title": "", + "description": "Microsoft Private Client API Credentials for connecting to Bing Ads", "required": [ "auth_method", - "developer_token", "client_id", "client_secret", "refresh_token" @@ -85,45 +71,45 @@ "const": "private_client", "order": 0 }, - "developer_token": { - "type": "string", - "title": "Developer Token", - "description": "Developer token associated with user. See more info in the docs.", - "airbyte_secret": true, - "order": 1 - }, "client_id": { "type": "string", "title": "Client ID", "description": "The Client ID of your Microsoft Advertising developer application.", "airbyte_secret": true, - "order": 2 + "order": 1 }, "client_secret": { "type": "string", "title": "Client Secret", "description": "The Client Secret of your Microsoft Advertising developer application.", "airbyte_secret": true, - "order": 3 + "order": 2 }, "refresh_token": { "type": "string", "title": "Refresh Token", "description": "Refresh Token to renew the expired Access Token.", "airbyte_secret": true, - "order": 4 + "order": 3 } } } ] }, + "developer_token": { + "type": "string", + "title": "Developer Token", + "description": "Developer token associated with user. See more info in the docs.", + "airbyte_secret": true, + "order": 2 + }, "reports_start_date": { "type": "string", "title": "Reports replication start date", "format": "date", "default": "2020-01-01", "description": "The start date from which to begin replicating report data. Any data generated before this date will not be replicated in reports. This is a UTC date in YYYY-MM-DD format.", - "order": 2 + "order": 3 }, "hourly_reports": { "title": "Enable hourly-aggregate reports", @@ -172,9 +158,6 @@ "properties": { "client_id": { "type": "string" - }, - "client_secret": { - "type": "string" } } }, @@ -185,10 +168,6 @@ "client_id": { "type": "string", "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] } } }, From 17143c0ea4b1eec1485cb1cc9417d27732512072 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Mon, 23 May 2022 18:33:45 +0300 Subject: [PATCH 17/19] updated after review --- .../integration_tests/invalid_config.json | 9 +- .../source-bing-ads/source_bing_ads/client.py | 42 +++---- .../source-bing-ads/source_bing_ads/spec.json | 117 ++++++------------ 3 files changed, 57 insertions(+), 111 deletions(-) diff --git a/airbyte-integrations/connectors/source-bing-ads/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-bing-ads/integration_tests/invalid_config.json index 74f21b9ab9e6..078e86a02384 100644 --- a/airbyte-integrations/connectors/source-bing-ads/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-bing-ads/integration_tests/invalid_config.json @@ -1,10 +1,7 @@ { - "credentials": { - "auth_method": "oauth2.0", - "refresh_token": "as2Ggas23gsa236gasgaskjfhas7i8ygf78as7osa7gy87asg8as7tg6as", - "client_secret": "1234", - "client_id": "123" - }, + "refresh_token": "as2Ggas23gsa236gasgaskjfhas7i8ygf78as7osa7gy87asg8as7tg6as", + "client_secret": "", + "client_id": "123", "developer_token": "asgag4gwag3", "reports_start_date": "2018-11-13", "hourly_reports": false, diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py index 1f9242f94829..82bf40fe329c 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py @@ -42,51 +42,39 @@ def __init__( daily_reports: bool, weekly_reports: bool, monthly_reports: bool, - credentials: dict = None, developer_token: str = None, - client_id: str = None, # deprecated - client_secret: str = None, # deprecated - refresh_token: str = None, # deprecated + client_id: str = None, + client_secret: str = None, + refresh_token: str = None, **kwargs: Mapping[str, Any], ) -> None: self.authorization_data: Mapping[str, AuthorizationData] = {} - self.refresh_token = credentials["refresh_token"] if credentials else refresh_token + self.refresh_token = refresh_token self.developer_token = developer_token self.hourly_reports = hourly_reports self.daily_reports = daily_reports self.weekly_reports = weekly_reports self.monthly_reports = monthly_reports - self.client_id = client_id # deprecated - self.client_secret = client_secret # deprecated + self.client_id = client_id + self.client_secret = client_secret - self.authentication = self._get_auth_client(credentials, tenant_id) + self.authentication = self._get_auth_client(client_id, tenant_id, client_secret) self.oauth: OAuthTokens = self._get_access_token() self.reports_start_date = pendulum.parse(reports_start_date).astimezone(tz=timezone.utc) - def _get_auth_client(self, credentials: dict, tenant_id: str) -> OAuthWebAuthCodeGrant: - - # base auth params + def _get_auth_client(self, client_id: str, tenant_id: str, client_secret: str = None) -> OAuthWebAuthCodeGrant: # https://github.com/BingAds/BingAds-Python-SDK/blob/e7b5a618e87a43d0a5e2c79d9aa4626e208797bd/bingads/authorization.py#L390 auth_creds = { - "client_id": None, - "client_secret": None, + "client_id": client_id, "redirection_uri": "", # should be empty string + "client_secret": None, "tenant": tenant_id, - } - - # support the deprecated old input configuration - if self.client_id or self.client_secret: - auth_creds["client_id"] = self.client_id - auth_creds["client_secret"] = self.client_secret - return OAuthWebAuthCodeGrant(**auth_creds) - - auth_creds["client_id"] = credentials["client_id"] - - if credentials["auth_method"] == "private_client": - # the `client_secret` should be provided for `non-public clients` ONLY - # https://docs.microsoft.com/en-us/advertising/guides/authentication-oauth-get-tokens?view=bingads-13#request-accesstoken - auth_creds["client_secret"] = credentials["client_secret"] + } + # the `client_secret` should be provided for `non-public clients` only + # https://docs.microsoft.com/en-us/advertising/guides/authentication-oauth-get-tokens?view=bingads-13#request-accesstoken + if client_secret and client_secret != "": + auth_creds["client_secret"] = client_secret return OAuthWebAuthCodeGrant(**auth_creds) @lru_cache(maxsize=None) diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json index 00e585df1ce4..c6c847e87703 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json @@ -6,6 +6,8 @@ "type": "object", "required": [ "developer_token", + "client_id", + "refresh_token", "reports_start_date", "hourly_reports", "daily_reports", @@ -14,6 +16,10 @@ ], "additionalProperties": true, "properties": { + "auth_method": { + "type": "string", + "const": "oauth2.0" + }, "tenant_id": { "type": "string", "title": "Tenant ID", @@ -22,86 +28,34 @@ "default": "common", "order": 0 }, - "credentials": { - "type": "object", - "title": "Authentication Method", - "description": "The authentication method to use to retrieve data from Microsoft Bing Ads", - "order": 1, - "oneOf": [ - { - "type": "object", - "title": "OAuth2.0", - "description": "Microsoft API Credentials for connecting to Bing Ads", - "required": ["auth_method"], - "properties": { - "auth_method": { - "type": "string", - "const": "oauth2.0", - "order": 0 - }, - "client_id": { - "type": "string", - "title": "Client ID", - "description": "The Client ID of your Microsoft Advertising developer application.", - "airbyte_secret": true, - "order": 1 - }, - "refresh_token": { - "type": "string", - "title": "Refresh Token", - "description": "Refresh Token to renew the expired Access Token.", - "airbyte_secret": true, - "order": 2 - } - } - }, - { - "type": "object", - "title": "", - "description": "Microsoft Private Client API Credentials for connecting to Bing Ads", - "required": [ - "auth_method", - "client_id", - "client_secret", - "refresh_token" - ], - "properties": { - "auth_method": { - "type": "string", - "const": "private_client", - "order": 0 - }, - "client_id": { - "type": "string", - "title": "Client ID", - "description": "The Client ID of your Microsoft Advertising developer application.", - "airbyte_secret": true, - "order": 1 - }, - "client_secret": { - "type": "string", - "title": "Client Secret", - "description": "The Client Secret of your Microsoft Advertising developer application.", - "airbyte_secret": true, - "order": 2 - }, - "refresh_token": { - "type": "string", - "title": "Refresh Token", - "description": "Refresh Token to renew the expired Access Token.", - "airbyte_secret": true, - "order": 3 - } - } - } - ] + "client_id": { + "type": "string", + "title": "Client ID", + "description": "The Client ID of your Microsoft Advertising developer application.", + "airbyte_secret": true, + "order": 1 + }, + "client_secret": { + "type": "string", + "title": "Client Secret", + "description": "The Client Secret of your Microsoft Advertising developer application.", + "default": "", + "airbyte_secret": true, + "order": 2 + }, + "refresh_token": { + "type": "string", + "title": "Refresh Token", + "description": "Refresh Token to renew the expired Access Token.", + "airbyte_secret": true, + "order": 3 }, "developer_token": { "type": "string", "title": "Developer Token", "description": "Developer token associated with user. See more info in the docs.", "airbyte_secret": true, - "order": 2 + "order": 4 }, "reports_start_date": { "type": "string", @@ -109,7 +63,7 @@ "format": "date", "default": "2020-01-01", "description": "The start date from which to begin replicating report data. Any data generated before this date will not be replicated in reports. This is a UTC date in YYYY-MM-DD format.", - "order": 3 + "order": 5 }, "hourly_reports": { "title": "Enable hourly-aggregate reports", @@ -139,7 +93,7 @@ }, "advanced_auth": { "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_method"], + "predicate_key": ["auth_method"], "predicate_value": "oauth2.0", "oauth_config_specification": { "complete_oauth_output_specification": { @@ -148,7 +102,7 @@ "properties": { "refresh_token": { "type": "string", - "path_in_connector_config": ["credentials", "refresh_token"] + "path_in_connector_config": ["refresh_token"] } } }, @@ -158,6 +112,9 @@ "properties": { "client_id": { "type": "string" + }, + "client_secret": { + "type": "string" } } }, @@ -167,7 +124,11 @@ "properties": { "client_id": { "type": "string", - "path_in_connector_config": ["credentials", "client_id"] + "path_in_connector_config": ["client_id"] + }, + "client_secret": { + "type": "string", + "path_in_connector_config": ["client_secret"] } } }, From 494341fc7b78377272c4714ef13b4823a6cf3abf Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Mon, 23 May 2022 20:00:02 +0300 Subject: [PATCH 18/19] updated docs --- docs/integrations/sources/bing-ads.md | 133 ++++++++++++++++---------- 1 file changed, 85 insertions(+), 48 deletions(-) diff --git a/docs/integrations/sources/bing-ads.md b/docs/integrations/sources/bing-ads.md index 0dc399c8fc5c..5eb509dab214 100644 --- a/docs/integrations/sources/bing-ads.md +++ b/docs/integrations/sources/bing-ads.md @@ -1,62 +1,99 @@ # Bing Ads -## Overview - -The Bing Ads connector syncs data from the [Bing Ads API](https://docs.microsoft.com/en-us/advertising/guides/?view=bingads-13). - -## Output schema - -This Source is capable of syncing the following resources: - -* [Accounts](https://docs.microsoft.com/en-us/advertising/customer-management-service/searchaccounts?view=bingads-13) -* [Campaigns](https://docs.microsoft.com/en-us/advertising/campaign-management-service/getcampaignsbyaccountid?view=bingads-13) -* [AdGroups](https://docs.microsoft.com/en-us/advertising/campaign-management-service/getadgroupsbycampaignid?view=bingads-13) -* [Ads](https://docs.microsoft.com/en-us/advertising/campaign-management-service/getadsbyadgroupid?view=bingads-13) - -It can also sync the following reports: - -* [AccountPerformanceReport](https://docs.microsoft.com/en-us/advertising/reporting-service/accountperformancereportrequest?view=bingads-13) -* [AdPerformanceReport](https://docs.microsoft.com/en-us/advertising/reporting-service/adperformancereportrequest?view=bingads-13) -* [AdGroupPerformanceReport](https://docs.microsoft.com/en-us/advertising/reporting-service/adgroupperformancereportrequest?view=bingads-13) -* [CampaignPerformanceReport](https://docs.microsoft.com/en-us/advertising/reporting-service/campaignperformancereportrequest?view=bingads-13) -* [BudgetSummaryReport](https://docs.microsoft.com/en-us/advertising/reporting-service/budgetsummaryreportrequest?view=bingads-13) -* [KeywordPerformanceReport](https://docs.microsoft.com/en-us/advertising/reporting-service/keywordperformancereportrequest?view=bingads-13) +This page guides you through the process of setting up the Bing Ads source connector. + +## Prerequisites (Airbyte Cloud) +* A Bing Ads account with permission to access data from accounts you want to sync + +## Prerequisites (Airbyte Open Source) +* Tenant ID +* Developer Token +* Client ID +* Client Secret +* Refresh Token +* Reports replication start date + +## Step 1: Set up Bing Ads + +1. Create a developer application using the instructions for [registering an application](https://docs.microsoft.com/en-us/advertising/guides/authentication-oauth-register?view=bingads-13) in Azure portal +2. Perform [these steps](https://docs.microsoft.com/en-us/advertising/guides/authentication-oauth-consent?view=bingads-13l) to get auth code, and use that to [get a refresh token](https://docs.microsoft.com/en-us/advertising/guides/authentication-oauth-get-tokens?view=bingads-13). For reference, the full authentication process described [here](https://docs.microsoft.com/en-us/advertising/guides/get-started?view=bingads-13#access-token). Be aware that the refresh token will expire in 90 days. You need to repeat the auth process to get a new refresh token. +3. Find your Microsoft developer token by following [these instructions](https://docs.microsoft.com/en-us/advertising/guides/get-started?view=bingads-13#get-developer-token) +4. Optionally, if your oauth app lives under a custom tenant which cannot use Microsoft's recommended `common` tenant, make sure to get the tenant ID ready for input when configuring the connector. The tenant will be used in the auth URL e.g: `https://login.microsoftonline.com//oauth2/v2.0/authorize`. + +## Step 2: Set up the source connector in Airbyte + +**For Airbyte Cloud:** + +1. [Log into your Airbyte Cloud](https://cloud.airbyte.io/workspaces) account. +2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**. +3. On the source setup page, select **Bing Ads** from the Source type dropdown and enter a name for this connector. +4. Add Tenant ID +5. Click `Authenticate your account`. +6. Log in and Authorize to the BingAds account +7. Choose required Start date and type of aggregation report +8. click `Set up source`. + +**For Airbyte OSS:** + +1. Go to local Airbyte page. +2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**. +3. On the Set up the source page, enter the name for the connector and select **Bing Ads** from the Source type dropdown. +4. Add Tenant ID +5. Copy and paste info from step 1: + * client ID + * client secret + * refresh_token + * A developer token +7. Choose required Start date and type of aggregation report +8. Click `Set up source`. + +## Supported sync modes + +The Bing Ads source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): + - Full Refresh + - Incremental + +## Supported Streams +Basic streams: +* [accounts](https://docs.microsoft.com/en-us/advertising/customer-management-service/searchaccounts?view=bingads-13) +* [campaigns](https://docs.microsoft.com/en-us/advertising/campaign-management-service/getcampaignsbyaccountid?view=bingads-13) +* [ad_groups](https://docs.microsoft.com/en-us/advertising/campaign-management-service/getadgroupsbycampaignid?view=bingads-13) +* [ads](https://docs.microsoft.com/en-us/advertising/campaign-management-service/getadsbyadgroupid?view=bingads-13) + +Report Streams: +* [budget_summary_report](https://docs.microsoft.com/en-us/advertising/reporting-service/budgetsummaryreportrequest?view=bingads-13) +* [account_performance_report_hourly](https://docs.microsoft.com/en-us/advertising/reporting-service/accountperformancereportrequest?view=bingads-13) +* [account_performance_report_daily](https://docs.microsoft.com/en-us/advertising/reporting-service/accountperformancereportrequest?view=bingads-13) +* [account_performance_report_weekly](https://docs.microsoft.com/en-us/advertising/reporting-service/accountperformancereportrequest?view=bingads-13) +* [account_performance_report_monthly](https://docs.microsoft.com/en-us/advertising/reporting-service/accountperformancereportrequest?view=bingads-13) +* [ad_group_performance_report_hourly](https://docs.microsoft.com/en-us/advertising/reporting-service/adgroupperformancereportrequest?view=bingads-13) +* [ad_group_performance_report_daily](https://docs.microsoft.com/en-us/advertising/reporting-service/adgroupperformancereportrequest?view=bingads-13) +* [ad_group_performance_report_weekly](https://docs.microsoft.com/en-us/advertising/reporting-service/adgroupperformancereportrequest?view=bingads-13) +* [ad_group_performance_report_monthly](https://docs.microsoft.com/en-us/advertising/reporting-service/adgroupperformancereportrequest?view=bingads-13) +* [ad_performance_report_hourly](https://docs.microsoft.com/en-us/advertising/reporting-service/adperformancereportrequest?view=bingads-13) +* [ad_performance_report_daily](https://docs.microsoft.com/en-us/advertising/reporting-service/adperformancereportrequest?view=bingads-13) +* [ad_performance_report_weekly](https://docs.microsoft.com/en-us/advertising/reporting-service/adperformancereportrequest?view=bingads-13) +* [ad_performance_report_monthly](https://docs.microsoft.com/en-us/advertising/reporting-service/adperformancereportrequest?view=bingads-13) +* [campaign_performance_report_hourly](https://docs.microsoft.com/en-us/advertising/reporting-service/campaignperformancereportrequest?view=bingads-13) +* [campaign_performance_report_daily](https://docs.microsoft.com/en-us/advertising/reporting-service/campaignperformancereportrequest?view=bingads-13) +* [campaign_performance_report_weekly](https://docs.microsoft.com/en-us/advertising/reporting-service/campaignperformancereportrequest?view=bingads-13) +* [campaign_performance_report_monthly](https://docs.microsoft.com/en-us/advertising/reporting-service/campaignperformancereportrequest?view=bingads-13) +* [keyword_performance_report_hourly](https://docs.microsoft.com/en-us/advertising/reporting-service/keywordperformancereportrequest?view=bingads-13) +* [keyword_performance_report_daily](https://docs.microsoft.com/en-us/advertising/reporting-service/keywordperformancereportrequest?view=bingads-13) +* [keyword_performance_report_weekly](https://docs.microsoft.com/en-us/advertising/reporting-service/keywordperformancereportrequest?view=bingads-13) +* [keyword_performance_report_monthly](https://docs.microsoft.com/en-us/advertising/reporting-service/keywordperformancereportrequest?view=bingads-13) + +For more information, see the [Bing Ads API](https://docs.microsoft.com/en-us/advertising/guides/?view=bingads-13). ### Report Aggregation All reports synced by this connector can be aggregated using hourly, daily, weekly, or monthly windows. Performance data is aggregated using the selected window. For example, if you select the daily-aggregation flavor of a report, the report will contain a row for each day for the duration of the report. Each row will indicate the number of impressions recorded on that day. -A report's aggregation window is indicated in its name e.g: `account_performance_report_hourly` is the Account Performance Reported aggregated using an hourly window. - -### Features - -| Feature | Supported?\(Yes/No\) | Notes | -| :--- |:---------------------| :--- | -| Full Refresh Sync | Yes | | -| Incremental Sync | Yes | | -| Namespaces | No | | +A report's aggregation window is indicated in its name e.g: `account_performance_report_hourly` is the Account Performance Reported aggregated using an hourly window. ### Performance considerations API limits number of requests for all Microsoft Advertising clients. You can find detailied info [here](https://docs.microsoft.com/en-us/advertising/guides/services-protocol?view=bingads-13#throttling) -## Getting started (Airbyte Open Source) -### Requirements -* A developer application with access to: - * client ID - * client secret - * A developer token - * Optionally, a tenant ID -* A refresh token generated using the above developer application credentials -* (Optional) Ad Account IDs you want to access, if you want to limit replication to specific ad accounts - -### Setup Guide -* Create a developer application using the instructions for [registering an application](https://docs.microsoft.com/en-us/advertising/guides/authentication-oauth-register?view=bingads-13) in Azure portal -* Perform [these steps](https://docs.microsoft.com/en-us/advertising/guides/authentication-oauth-consent?view=bingads-13l) to get auth code, and use that to [get a refresh token](https://docs.microsoft.com/en-us/advertising/guides/authentication-oauth-get-tokens?view=bingads-13). For reference, the full authentication process described [here](https://docs.microsoft.com/en-us/advertising/guides/get-started?view=bingads-13#access-token). Be aware that the refresh token will expire in 90 days. You need to repeat the auth process to get a new refresh token. -* Find your Microsoft developer token by following [these instructions](https://docs.microsoft.com/en-us/advertising/guides/get-started?view=bingads-13#get-developer-token) -* Optionally, if your oauth app lives under a custom tenant which cannot use Microsoft's recommended `common` tenant, make sure to get the tenant ID ready for input when configuring the connector. The tenant will be used in the auth URL e.g: `https://login.microsoftonline.com//oauth2/v2.0/authorize`. - - - ## Changelog | Version | Date | Pull Request | Subject | From f82494cd6e2d72746fe0a98f6e91ebec80b19156 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Mon, 23 May 2022 18:05:10 +0000 Subject: [PATCH 19/19] auto-bump connector version --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 138 ++++++++---------- 2 files changed, 65 insertions(+), 75 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index f9463dfb3916..ba0188b07c33 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -104,7 +104,7 @@ - name: Bing Ads sourceDefinitionId: 47f25999-dd5e-4636-8c39-e7cea2453331 dockerRepository: airbyte/source-bing-ads - dockerImageTag: 0.1.6 + dockerImageTag: 0.1.7 documentationUrl: https://docs.airbyte.io/integrations/sources/bing-ads icon: bingads.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 71edc9509a4f..aa9307e01711 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -801,7 +801,7 @@ - "overwrite" - "append" - "append_dedup" -- dockerImage: "airbyte/source-bing-ads:0.1.6" +- dockerImage: "airbyte/source-bing-ads:0.1.7" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/bing-ads" connectionSpecification: @@ -809,107 +809,55 @@ title: "Bing Ads Spec" type: "object" required: - - "accounts" - - "client_id" - - "client_secret" - - "customer_id" - "developer_token" + - "client_id" - "refresh_token" - - "user_id" - "reports_start_date" - "hourly_reports" - "daily_reports" - "weekly_reports" - "monthly_reports" - additionalProperties: false + additionalProperties: true properties: - accounts: - title: "Accounts to replicate data from" - type: "object" - description: "" - oneOf: - - title: "All Accounts" - additionalProperties: false - description: "Replicate data from all accounts to which you have access." - required: - - "selection_strategy" - properties: - selection_strategy: - type: "string" - const: "all" - - title: "Specific Accounts" - additionalProperties: false - description: "Fetch data for subset of account IDs." - required: - - "ids" - - "selection_strategy" - properties: - selection_strategy: - type: "string" - const: "subset" - ids: - type: "array" - title: "Account IDs" - description: "List of the account IDs from which data will be replicated." - items: - type: "string" - minItems: 1 - uniqueItems: true + auth_method: + type: "string" + const: "oauth2.0" + tenant_id: + type: "string" + title: "Tenant ID" + description: "The Tenant ID of your Microsoft Advertising developer application.\ + \ Set this to \"common\" unless you know you need a different value." + airbyte_secret: true + default: "common" + order: 0 client_id: type: "string" title: "Client ID" description: "The Client ID of your Microsoft Advertising developer application." airbyte_secret: true - order: 0 + order: 1 client_secret: type: "string" title: "Client Secret" description: "The Client Secret of your Microsoft Advertising developer\ \ application." + default: "" airbyte_secret: true - order: 1 + order: 2 refresh_token: type: "string" title: "Refresh Token" description: "Refresh Token to renew the expired Access Token." airbyte_secret: true - order: 2 + order: 3 developer_token: type: "string" title: "Developer Token" - description: "Developer token associated with user." + description: "Developer token associated with user. See more info in the docs." airbyte_secret: true - order: 3 - tenant_id: - type: "string" - title: "Tenant ID" - description: "The Tenant ID of your Microsoft Advertising developer application.\ - \ Set this to \"common\" unless you know you need a different value." - airbyte_secret: true - default: "common" order: 4 - redirect_uri: - type: "string" - title: "Redirect URI (Optional)" - description: "The Redirect URI of your Microsoft Advertising developer application.\ - \ Leave this empty unless you know that you need it." - airbyte_secret: true - default: "" - order: 5 - customer_id: - type: "string" - title: "Customer ID" - description: "Your Bing Customer ID. See the \"Getting Started\" section\ - \ in the docs for information on how to obtain this ID" - order: 6 - user_id: - type: "string" - title: "Account ID" - description: "Bing Ads Account ID. See the \"Getting Started\" section in\ - \ the\ - \ docs for information on how to obtain this ID" - order: 7 reports_start_date: type: "string" title: "Reports replication start date" @@ -918,7 +866,7 @@ description: "The start date from which to begin replicating report data.\ \ Any data generated before this date will not be replicated in reports.\ \ This is a UTC date in YYYY-MM-DD format." - order: 8 + order: 5 hourly_reports: title: "Enable hourly-aggregate reports" type: "boolean" @@ -954,6 +902,48 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] + advanced_auth: + auth_flow_type: "oauth2.0" + predicate_key: + - "auth_method" + predicate_value: "oauth2.0" + oauth_config_specification: + oauth_user_input_from_connector_config_specification: + type: "object" + additionalProperties: false + properties: + tenant_id: + type: "string" + path_in_connector_config: + - "tenant_id" + complete_oauth_output_specification: + type: "object" + additionalProperties: false + properties: + refresh_token: + type: "string" + path_in_connector_config: + - "refresh_token" + complete_oauth_server_input_specification: + type: "object" + additionalProperties: false + properties: + client_id: + type: "string" + client_secret: + type: "string" + complete_oauth_server_output_specification: + type: "object" + additionalProperties: false + properties: + client_id: + type: "string" + path_in_connector_config: + - "client_id" + client_secret: + type: "string" + path_in_connector_config: + - "client_secret" - dockerImage: "airbyte/source-braintree:0.1.3" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/braintree"