diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index eef41b0747ffc..750e496882656 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -104,11 +104,11 @@ - name: Bing Ads sourceDefinitionId: 47f25999-dd5e-4636-8c39-e7cea2453331 dockerRepository: airbyte/source-bing-ads - dockerImageTag: 0.1.6 + dockerImageTag: 0.1.7 documentationUrl: https://docs.airbyte.io/integrations/sources/bing-ads icon: bingads.svg sourceType: api - releaseStage: alpha + releaseStage: beta - name: Braintree sourceDefinitionId: 63cea06f-1c75-458d-88fe-ad48c7cb27fd dockerRepository: airbyte/source-braintree diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 96aa2c2d9b523..53fa5f1cdde3b 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -801,7 +801,7 @@ - "overwrite" - "append" - "append_dedup" -- dockerImage: "airbyte/source-bing-ads:0.1.6" +- dockerImage: "airbyte/source-bing-ads:0.1.7" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/bing-ads" connectionSpecification: @@ -809,107 +809,55 @@ title: "Bing Ads Spec" type: "object" required: - - "accounts" - - "client_id" - - "client_secret" - - "customer_id" - "developer_token" + - "client_id" - "refresh_token" - - "user_id" - "reports_start_date" - "hourly_reports" - "daily_reports" - "weekly_reports" - "monthly_reports" - additionalProperties: false + additionalProperties: true properties: - accounts: - title: "Accounts to replicate data from" - type: "object" - description: "" - oneOf: - - title: "All Accounts" - additionalProperties: false - description: "Replicate data from all accounts to which you have access." - required: - - "selection_strategy" - properties: - selection_strategy: - type: "string" - const: "all" - - title: "Specific Accounts" - additionalProperties: false - description: "Fetch data for subset of account IDs." - required: - - "ids" - - "selection_strategy" - properties: - selection_strategy: - type: "string" - const: "subset" - ids: - type: "array" - title: "Account IDs" - description: "List of the account IDs from which data will be replicated." - items: - type: "string" - minItems: 1 - uniqueItems: true + auth_method: + type: "string" + const: "oauth2.0" + tenant_id: + type: "string" + title: "Tenant ID" + description: "The Tenant ID of your Microsoft Advertising developer application.\ + \ Set this to \"common\" unless you know you need a different value." + airbyte_secret: true + default: "common" + order: 0 client_id: type: "string" title: "Client ID" description: "The Client ID of your Microsoft Advertising developer application." airbyte_secret: true - order: 0 + order: 1 client_secret: type: "string" title: "Client Secret" description: "The Client Secret of your Microsoft Advertising developer\ \ application." + default: "" airbyte_secret: true - order: 1 + order: 2 refresh_token: type: "string" title: "Refresh Token" description: "Refresh Token to renew the expired Access Token." airbyte_secret: true - order: 2 + order: 3 developer_token: type: "string" title: "Developer Token" - description: "Developer token associated with user." + description: "Developer token associated with user. See more info in the docs." airbyte_secret: true - order: 3 - tenant_id: - type: "string" - title: "Tenant ID" - description: "The Tenant ID of your Microsoft Advertising developer application.\ - \ Set this to \"common\" unless you know you need a different value." - airbyte_secret: true - default: "common" order: 4 - redirect_uri: - type: "string" - title: "Redirect URI (Optional)" - description: "The Redirect URI of your Microsoft Advertising developer application.\ - \ Leave this empty unless you know that you need it." - airbyte_secret: true - default: "" - order: 5 - customer_id: - type: "string" - title: "Customer ID" - description: "Your Bing Customer ID. See the \"Getting Started\" section\ - \ in the docs for information on how to obtain this ID" - order: 6 - user_id: - type: "string" - title: "Account ID" - description: "Bing Ads Account ID. See the \"Getting Started\" section in\ - \ the\ - \ docs for information on how to obtain this ID" - order: 7 reports_start_date: type: "string" title: "Reports replication start date" @@ -918,7 +866,7 @@ description: "The start date from which to begin replicating report data.\ \ Any data generated before this date will not be replicated in reports.\ \ This is a UTC date in YYYY-MM-DD format." - order: 8 + order: 5 hourly_reports: title: "Enable hourly-aggregate reports" type: "boolean" @@ -954,6 +902,48 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] + advanced_auth: + auth_flow_type: "oauth2.0" + predicate_key: + - "auth_method" + predicate_value: "oauth2.0" + oauth_config_specification: + oauth_user_input_from_connector_config_specification: + type: "object" + additionalProperties: false + properties: + tenant_id: + type: "string" + path_in_connector_config: + - "tenant_id" + complete_oauth_output_specification: + type: "object" + additionalProperties: false + properties: + refresh_token: + type: "string" + path_in_connector_config: + - "refresh_token" + complete_oauth_server_input_specification: + type: "object" + additionalProperties: false + properties: + client_id: + type: "string" + client_secret: + type: "string" + complete_oauth_server_output_specification: + type: "object" + additionalProperties: false + properties: + client_id: + type: "string" + path_in_connector_config: + - "client_id" + client_secret: + type: "string" + path_in_connector_config: + - "client_secret" - dockerImage: "airbyte/source-braintree:0.1.3" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/braintree" diff --git a/airbyte-integrations/connectors/source-bing-ads/Dockerfile b/airbyte-integrations/connectors/source-bing-ads/Dockerfile index 8349748f3000d..683c8ee290447 100644 --- a/airbyte-integrations/connectors/source-bing-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-bing-ads/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.6 +LABEL io.airbyte.version=0.1.7 LABEL io.airbyte.name=airbyte/source-bing-ads diff --git a/airbyte-integrations/connectors/source-bing-ads/README.md b/airbyte-integrations/connectors/source-bing-ads/README.md index 598e2be036ccf..22a0fc0fa7f1a 100644 --- a/airbyte-integrations/connectors/source-bing-ads/README.md +++ b/airbyte-integrations/connectors/source-bing-ads/README.md @@ -116,7 +116,8 @@ Customize `acceptance-test-config.yml` file to configure tests. See [Source Acce If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. To run your integration tests with acceptance tests, from the connector root, run ``` -python -m pytest integration_tests -p integration_tests.acceptance +docker build . --no-cache -t airbyte/source-bing-ads:dev \ +&& python -m pytest -p source_acceptance_test.plugin ``` To run your integration tests with docker diff --git a/airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml index 205bc72cbedd4..c73cbe55e46ca 100644 --- a/airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml @@ -3,6 +3,8 @@ tests: spec: - spec_path: "source_bing_ads/spec.json" connection: + - config_path: "secrets/config_old.json" + status: "succeed" - config_path: "secrets/config.json" status: "succeed" - config_path: "integration_tests/invalid_config.json" diff --git a/airbyte-integrations/connectors/source-bing-ads/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-bing-ads/integration_tests/invalid_config.json index af0ab21003f5d..078e86a023842 100644 --- a/airbyte-integrations/connectors/source-bing-ads/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-bing-ads/integration_tests/invalid_config.json @@ -1,16 +1,12 @@ { - "accounts": { "selection_strategy": "all" }, - "user_id": "2222", - "customer_id": "1111", - "developer_token": "asgag4gwag3", "refresh_token": "as2Ggas23gsa236gasgaskjfhas7i8ygf78as7osa7gy87asg8as7tg6as", - "client_secret": "1234", + "client_secret": "", "client_id": "123", - "tenant_id": "common", - "redirect_uri": "", + "developer_token": "asgag4gwag3", "reports_start_date": "2018-11-13", - "hourly_reports": true, + "hourly_reports": false, "daily_reports": false, - "weekly_reports": false, - "monthly_reports": true + "weekly_reports": true, + "monthly_reports": true, + "tenant_id": "common" } diff --git a/airbyte-integrations/connectors/source-bing-ads/setup.py b/airbyte-integrations/connectors/source-bing-ads/setup.py index 1c86a5ade1b37..897671c508de5 100644 --- a/airbyte-integrations/connectors/source-bing-ads/setup.py +++ b/airbyte-integrations/connectors/source-bing-ads/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk", "bingads~=13.0.11", "vcrpy==4.1.1", "backoff==1.10.0", "pendulum==2.1.2"] +MAIN_REQUIREMENTS = ["airbyte-cdk", "bingads~=13.0.13", "vcrpy==4.1.1", "backoff==1.10.0", "pendulum==2.1.2"] TEST_REQUIREMENTS = [ "pytest~=6.1", diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py index 441a144617fdc..82bf40fe329c3 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py @@ -36,44 +36,52 @@ class Client: def __init__( self, - developer_token: str, - customer_id: str, - client_secret: str, - client_id: str, tenant_id: str, - redirect_uri: str, - refresh_token: str, reports_start_date: str, hourly_reports: bool, daily_reports: bool, weekly_reports: bool, monthly_reports: bool, + developer_token: str = None, + client_id: str = None, + client_secret: str = None, + refresh_token: str = None, **kwargs: Mapping[str, Any], ) -> None: self.authorization_data: Mapping[str, AuthorizationData] = {} - self.authentication = OAuthWebAuthCodeGrant( - client_id, - client_secret, - redirect_uri, - tenant=tenant_id, - ) - self.refresh_token = refresh_token - self.customer_id = customer_id self.developer_token = developer_token self.hourly_reports = hourly_reports self.daily_reports = daily_reports self.weekly_reports = weekly_reports self.monthly_reports = monthly_reports + self.client_id = client_id + self.client_secret = client_secret + + self.authentication = self._get_auth_client(client_id, tenant_id, client_secret) self.oauth: OAuthTokens = self._get_access_token() self.reports_start_date = pendulum.parse(reports_start_date).astimezone(tz=timezone.utc) + def _get_auth_client(self, client_id: str, tenant_id: str, client_secret: str = None) -> OAuthWebAuthCodeGrant: + # https://github.com/BingAds/BingAds-Python-SDK/blob/e7b5a618e87a43d0a5e2c79d9aa4626e208797bd/bingads/authorization.py#L390 + auth_creds = { + "client_id": client_id, + "redirection_uri": "", # should be empty string + "client_secret": None, + "tenant": tenant_id, + } + # the `client_secret` should be provided for `non-public clients` only + # https://docs.microsoft.com/en-us/advertising/guides/authentication-oauth-get-tokens?view=bingads-13#request-accesstoken + if client_secret and client_secret != "": + auth_creds["client_secret"] = client_secret + return OAuthWebAuthCodeGrant(**auth_creds) + @lru_cache(maxsize=None) - def _get_auth_data(self, account_id: Optional[str] = None) -> AuthorizationData: + def _get_auth_data(self, customer_id: str = None, account_id: Optional[str] = None) -> AuthorizationData: return AuthorizationData( account_id=account_id, - customer_id=self.customer_id, + customer_id=customer_id, developer_token=self.developer_token, authentication=self.authentication, ) @@ -124,6 +132,7 @@ def _request( self, service_name: Optional[str], operation_name: str, + customer_id: Optional[str], account_id: Optional[str], params: Mapping[str, Any], is_report_service: bool = False, @@ -135,9 +144,9 @@ def _request( self.oauth = self._get_access_token() if is_report_service: - service = self._get_reporting_service(account_id=account_id) + service = self._get_reporting_service(customer_id=customer_id, account_id=account_id) else: - service = self.get_service(service_name=service_name, account_id=account_id) + service = self.get_service(service_name=service_name, customer_id=customer_id, account_id=account_id) return getattr(service, operation_name)(**params) @@ -145,22 +154,24 @@ def _request( def get_service( self, service_name: str, + customer_id: str = None, account_id: Optional[str] = None, ) -> ServiceClient: return ServiceClient( service=service_name, version=self.api_version, - authorization_data=self._get_auth_data(account_id), + authorization_data=self._get_auth_data(customer_id, account_id), environment=self.environment, ) @lru_cache(maxsize=None) def _get_reporting_service( self, + customer_id: Optional[str] = None, account_id: Optional[str] = None, ) -> ServiceClient: return ReportingServiceManager( - authorization_data=self._get_auth_data(account_id), + authorization_data=self._get_auth_data(customer_id, account_id), poll_interval_in_milliseconds=self.report_poll_interval, environment=self.environment, ) diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py index a872c66ff0a10..b6af1070d4c02 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py @@ -170,9 +170,10 @@ def get_updated_state( ) return current_stream_state - def send_request(self, params: Mapping[str, Any], account_id: str) -> _RowReport: + def send_request(self, params: Mapping[str, Any], customer_id: str, account_id: str) -> _RowReport: request_kwargs = { "service_name": None, + "customer_id": customer_id, "account_id": account_id, "operation_name": self.operation_name, "is_report_service": True, @@ -262,6 +263,6 @@ def stream_slices( **kwargs: Mapping[str, Any], ) -> Iterable[Optional[Mapping[str, Any]]]: for account in source_bing_ads.source.Accounts(self.client, self.config).read_records(SyncMode.full_refresh): - yield {"account_id": account["Id"]} + yield {"account_id": account["Id"], "customer_id": account["ParentCustomerId"]} yield from [] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/accounts.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/accounts.json index fc5f671d061ed..edce04ef5679e 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/accounts.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/accounts.json @@ -80,7 +80,7 @@ "type": ["null", "number"] }, "PauseReason": { - "type": ["null", "string"] + "type": ["null", "number"] }, "PaymentMethodId": { "type": ["null", "number"] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py index eb6ae52d81bca..f8383a2439387 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py @@ -10,6 +10,8 @@ from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream +from bingads.service_client import ServiceClient +from bingads.v13.reporting.reporting_service_manager import ReportingServiceManager from source_bing_ads.cache import VcrCache from source_bing_ads.client import Client from source_bing_ads.reports import ReportsMixin @@ -61,6 +63,14 @@ def additional_fields(self) -> Optional[str]: """ pass + @property + def _service(self) -> Union[ServiceClient, ReportingServiceManager]: + return self.client.get_service(service_name=self.service_name) + + @property + def _user_id(self) -> int: + return self._service.GetUser().User.Id + def next_page_token(self, response: sudsobject.Object, **kwargs: Mapping[str, Any]) -> Optional[Mapping[str, Any]]: """ Default method for streams that don't support pagination @@ -73,24 +83,20 @@ def parse_response(self, response: sudsobject.Object, **kwargs) -> Iterable[Mapp yield from [] - def send_request(self, params: Mapping[str, Any], account_id: str = None) -> Mapping[str, Any]: + def send_request(self, params: Mapping[str, Any], customer_id: str, account_id: str = None) -> Mapping[str, Any]: request_kwargs = { "service_name": self.service_name, + "customer_id": customer_id, "account_id": account_id, "operation_name": self.operation_name, "params": params, } - if not self.use_cache: - return self.client.request(**request_kwargs) - - with CACHE.use_cassette(): - return self.client.request(**request_kwargs) - - def get_account_id(self, stream_slice: Mapping[str, Any] = None) -> Optional[str]: - """ - Fetches account_id from slice object - """ - return str(stream_slice.get("account_id")) if stream_slice else None + request = self.client.request(**request_kwargs) + if self.use_cache: + with CACHE.use_cassette(): + return request + else: + return request def read_records( self, @@ -101,14 +107,17 @@ def read_records( ) -> Iterable[Mapping[str, Any]]: stream_state = stream_state or {} next_page_token = None - account_id = self.get_account_id(stream_slice) + account_id = str(stream_slice.get("account_id")) if stream_slice else None + customer_id = str(stream_slice.get("customer_id")) if stream_slice else None while True: params = self.request_params( - stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token, account_id=account_id + stream_state=stream_state, + stream_slice=stream_slice, + next_page_token=next_page_token, + account_id=account_id, ) - - response = self.send_request(params, account_id=account_id) + response = self.send_request(params, customer_id=customer_id, account_id=account_id) for record in self.parse_response(response): yield record @@ -154,21 +163,12 @@ def request_params( { "Field": "UserId", "Operator": "Equals", - "Value": self.config["user_id"], + "Value": self._user_id, } ] } - if self.config["accounts"]["selection_strategy"] == "subset": - predicates["Predicate"].append( - { - "Field": "AccountId", - "Operator": "In", - "Value": ",".join(self.config["accounts"]["ids"]), - } - ) - - paging = self.client.get_service(service_name=self.service_name).factory.create("ns5:Paging") + paging = self._service.factory.create("ns5:Paging") paging.Index = next_page_token or 0 paging.Size = self.page_size_limit return { @@ -213,7 +213,7 @@ def stream_slices( **kwargs: Mapping[str, Any], ) -> Iterable[Optional[Mapping[str, Any]]]: for account in Accounts(self.client, self.config).read_records(SyncMode.full_refresh): - yield {"account_id": account["Id"]} + yield {"account_id": account["Id"], "customer_id": account["ParentCustomerId"]} yield from [] @@ -247,8 +247,10 @@ def stream_slices( ) -> Iterable[Optional[Mapping[str, Any]]]: campaigns = Campaigns(self.client, self.config) for account in Accounts(self.client, self.config).read_records(SyncMode.full_refresh): - for campaign in campaigns.read_records(sync_mode=SyncMode.full_refresh, stream_slice={"account_id": account["Id"]}): - yield {"campaign_id": campaign["Id"], "account_id": account["Id"]} + for campaign in campaigns.read_records( + sync_mode=SyncMode.full_refresh, stream_slice={"account_id": account["Id"], "customer_id": account["ParentCustomerId"]} + ): + yield {"campaign_id": campaign["Id"], "account_id": account["Id"], "customer_id": account["ParentCustomerId"]} yield from [] @@ -294,7 +296,7 @@ def stream_slices( ad_groups = AdGroups(self.client, self.config) for slice in ad_groups.stream_slices(sync_mode=SyncMode.full_refresh): for ad_group in ad_groups.read_records(sync_mode=SyncMode.full_refresh, stream_slice=slice): - yield {"ad_group_id": ad_group["Id"], "account_id": slice["account_id"]} + yield {"ad_group_id": ad_group["Id"], "account_id": slice["account_id"], "customer_id": slice["customer_id"]} yield from [] @@ -570,21 +572,13 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> try: client = Client(**config) account_ids = {str(account["Id"]) for account in Accounts(client, config).read_records(SyncMode.full_refresh)} - - if config["accounts"]["selection_strategy"] == "subset": - config_account_ids = set(config["accounts"]["ids"]) - if not config_account_ids.issubset(account_ids): - raise Exception(f"Accounts with ids: {config_account_ids.difference(account_ids)} not found on this user.") - elif config["accounts"]["selection_strategy"] == "all": - if not account_ids: - raise Exception("You don't have accounts assigned to this user.") + if account_ids: + return True, None else: - raise Exception("Incorrect account selection strategy.") + raise Exception("You don't have accounts assigned to this user.") except Exception as error: return False, error - return True, None - def get_report_streams(self, aggregation_type: str) -> List[Stream]: return [ globals()[f"AccountPerformanceReport{aggregation_type}"], diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json index d8dba81babfbb..c6c847e87703c 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json @@ -5,125 +5,65 @@ "title": "Bing Ads Spec", "type": "object", "required": [ - "accounts", - "client_id", - "client_secret", - "customer_id", "developer_token", + "client_id", "refresh_token", - "user_id", "reports_start_date", "hourly_reports", "daily_reports", "weekly_reports", "monthly_reports" ], - "additionalProperties": false, + "additionalProperties": true, "properties": { - "accounts": { - "title": "Accounts to replicate data from", - "type": "object", - "description": "", - "oneOf": [ - { - "title": "All Accounts", - "additionalProperties": false, - "description": "Replicate data from all accounts to which you have access.", - "required": ["selection_strategy"], - "properties": { - "selection_strategy": { - "type": "string", - "const": "all" - } - } - }, - { - "title": "Specific Accounts", - "additionalProperties": false, - "description": "Fetch data for subset of account IDs.", - "required": ["ids", "selection_strategy"], - "properties": { - "selection_strategy": { - "type": "string", - "const": "subset" - }, - "ids": { - "type": "array", - "title": "Account IDs", - "description": "List of the account IDs from which data will be replicated.", - "items": { - "type": "string" - }, - "minItems": 1, - "uniqueItems": true - } - } - } - ] + "auth_method": { + "type": "string", + "const": "oauth2.0" + }, + "tenant_id": { + "type": "string", + "title": "Tenant ID", + "description": "The Tenant ID of your Microsoft Advertising developer application. Set this to \"common\" unless you know you need a different value.", + "airbyte_secret": true, + "default": "common", + "order": 0 }, "client_id": { "type": "string", "title": "Client ID", "description": "The Client ID of your Microsoft Advertising developer application.", "airbyte_secret": true, - "order": 0 + "order": 1 }, "client_secret": { "type": "string", "title": "Client Secret", "description": "The Client Secret of your Microsoft Advertising developer application.", + "default": "", "airbyte_secret": true, - "order": 1 + "order": 2 }, "refresh_token": { "type": "string", "title": "Refresh Token", "description": "Refresh Token to renew the expired Access Token.", "airbyte_secret": true, - "order": 2 + "order": 3 }, "developer_token": { "type": "string", "title": "Developer Token", - "description": "Developer token associated with user.", - "airbyte_secret": true, - "order": 3 - }, - "tenant_id": { - "type": "string", - "title": "Tenant ID", - "description": "The Tenant ID of your Microsoft Advertising developer application. Set this to \"common\" unless you know you need a different value.", + "description": "Developer token associated with user. See more info in the docs.", "airbyte_secret": true, - "default": "common", "order": 4 }, - "redirect_uri": { - "type": "string", - "title": "Redirect URI (Optional)", - "description": "The Redirect URI of your Microsoft Advertising developer application. Leave this empty unless you know that you need it.", - "airbyte_secret": true, - "default": "", - "order": 5 - }, - "customer_id": { - "type": "string", - "title": "Customer ID", - "description": "Your Bing Customer ID. See the \"Getting Started\" section in the docs for information on how to obtain this ID", - "order": 6 - }, - "user_id": { - "type": "string", - "title": "Account ID", - "description": "Bing Ads Account ID. See the \"Getting Started\" section in the docs for information on how to obtain this ID", - "order": 7 - }, "reports_start_date": { "type": "string", "title": "Reports replication start date", "format": "date", "default": "2020-01-01", "description": "The start date from which to begin replicating report data. Any data generated before this date will not be replicated in reports. This is a UTC date in YYYY-MM-DD format.", - "order": 8 + "order": 5 }, "hourly_reports": { "title": "Enable hourly-aggregate reports", @@ -150,5 +90,58 @@ "default": false } } + }, + "advanced_auth": { + "auth_flow_type": "oauth2.0", + "predicate_key": ["auth_method"], + "predicate_value": "oauth2.0", + "oauth_config_specification": { + "complete_oauth_output_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "refresh_token": { + "type": "string", + "path_in_connector_config": ["refresh_token"] + } + } + }, + "complete_oauth_server_input_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + } + } + }, + "complete_oauth_server_output_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "client_id": { + "type": "string", + "path_in_connector_config": ["client_id"] + }, + "client_secret": { + "type": "string", + "path_in_connector_config": ["client_secret"] + } + } + }, + "oauth_user_input_from_connector_config_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "tenant_id": { + "type": "string", + "path_in_connector_config": ["tenant_id"] + } + } + } + } } } diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/accountperformancereportmonthly_records.json b/airbyte-integrations/connectors/source-bing-ads/unit_tests/accountperformancereportmonthly_records.json new file mode 100644 index 0000000000000..9f8ee44fbb6a0 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/accountperformancereportmonthly_records.json @@ -0,0 +1,354 @@ +[ + { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-06-01", + "DeviceType": "Computer", + "Network": "Bing and Yahoo! search", + "Impressions": 1380, + "Clicks": 22, + "Spend": 3.34, + "Ctr": 0.0159, + "AverageCpc": 0.15, + "ReturnOnAdSpend": 0.0, + "RevenuePerConversion": null, + "ConversionRate": null + }, + { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-06-01", + "DeviceType": "Computer", + "Network": "Syndicated search partners", + "Impressions": 1992, + "Clicks": 25, + "Spend": 4.62, + "Ctr": 0.0126, + "AverageCpc": 0.18, + "ReturnOnAdSpend": 0.0, + "RevenuePerConversion": null, + "ConversionRate": null + }, + { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-06-01", + "DeviceType": "Computer", + "Network": "AOL search", + "Impressions": 2, + "Clicks": 0, + "Spend": 0.0, + "Ctr": 0.0, + "AverageCpc": 0.0, + "ReturnOnAdSpend": null, + "RevenuePerConversion": null, + "ConversionRate": null + }, + { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-06-01", + "DeviceType": "Computer", + "Network": "Audience", + "Impressions": 9, + "Clicks": 0, + "Spend": 0.0, + "Ctr": 0.0, + "AverageCpc": 0.0, + "ReturnOnAdSpend": null, + "RevenuePerConversion": null, + "ConversionRate": null + }, + { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-06-01", + "DeviceType": "Smartphone", + "Network": "Bing and Yahoo! search", + "Impressions": 73, + "Clicks": 5, + "Spend": 0.98, + "Ctr": 0.06849999999999999, + "AverageCpc": 0.2, + "ReturnOnAdSpend": 0.0, + "RevenuePerConversion": null, + "ConversionRate": null + }, + { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-06-01", + "DeviceType": "Smartphone", + "Network": "Syndicated search partners", + "Impressions": 5754, + "Clicks": 38, + "Spend": 7.1, + "Ctr": 0.0066, + "AverageCpc": 0.19, + "ReturnOnAdSpend": 0.0, + "RevenuePerConversion": null, + "ConversionRate": null + }, + { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-06-01", + "DeviceType": "Tablet", + "Network": "Bing and Yahoo! search", + "Impressions": 1, + "Clicks": 0, + "Spend": 0.0, + "Ctr": 0.0, + "AverageCpc": 0.0, + "ReturnOnAdSpend": null, + "RevenuePerConversion": null, + "ConversionRate": null + }, + { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-06-01", + "DeviceType": "Tablet", + "Network": "Syndicated search partners", + "Impressions": 82, + "Clicks": 1, + "Spend": 0.22, + "Ctr": 0.012199999999999999, + "AverageCpc": 0.22, + "ReturnOnAdSpend": 0.0, + "RevenuePerConversion": null, + "ConversionRate": null + }, + { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-06-01", + "DeviceType": "Tablet", + "Network": "AOL search", + "Impressions": 0, + "Clicks": 0, + "Spend": 0.0, + "Ctr": null, + "AverageCpc": 0.0, + "ReturnOnAdSpend": null, + "RevenuePerConversion": null, + "ConversionRate": null + }, + { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-07-01", + "DeviceType": "Computer", + "Network": "Bing and Yahoo! search", + "Impressions": 783, + "Clicks": 12, + "Spend": 1.23, + "Ctr": 0.015300000000000001, + "AverageCpc": 0.1, + "ReturnOnAdSpend": 0.0, + "RevenuePerConversion": null, + "ConversionRate": null + }, + { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-07-01", + "DeviceType": "Computer", + "Network": "Syndicated search partners", + "Impressions": 1712, + "Clicks": 23, + "Spend": 2.72, + "Ctr": 0.0134, + "AverageCpc": 0.12, + "ReturnOnAdSpend": 0.0, + "RevenuePerConversion": null, + "ConversionRate": null + }, + { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-07-01", + "DeviceType": "Computer", + "Network": "AOL search", + "Impressions": 0, + "Clicks": 0, + "Spend": 0.0, + "Ctr": null, + "AverageCpc": 0.0, + "ReturnOnAdSpend": null, + "RevenuePerConversion": null, + "ConversionRate": null + }, + { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-07-01", + "DeviceType": "Computer", + "Network": "Audience", + "Impressions": 9, + "Clicks": 0, + "Spend": 0.0, + "Ctr": 0.0, + "AverageCpc": 0.0, + "ReturnOnAdSpend": null, + "RevenuePerConversion": null, + "ConversionRate": null + }, + { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-07-01", + "DeviceType": "Smartphone", + "Network": "Bing and Yahoo! search", + "Impressions": 52, + "Clicks": 3, + "Spend": 0.19, + "Ctr": 0.057699999999999994, + "AverageCpc": 0.06, + "ReturnOnAdSpend": 0.0, + "RevenuePerConversion": null, + "ConversionRate": null + }, + { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-07-01", + "DeviceType": "Smartphone", + "Network": "Syndicated search partners", + "Impressions": 38546, + "Clicks": 201, + "Spend": 18.1, + "Ctr": 0.0052, + "AverageCpc": 0.09, + "ReturnOnAdSpend": 0.0, + "RevenuePerConversion": null, + "ConversionRate": null + }, + { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-07-01", + "DeviceType": "Tablet", + "Network": "Bing and Yahoo! search", + "Impressions": 1, + "Clicks": 0, + "Spend": 0.0, + "Ctr": 0.0, + "AverageCpc": 0.0, + "ReturnOnAdSpend": null, + "RevenuePerConversion": null, + "ConversionRate": null + }, + { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-07-01", + "DeviceType": "Tablet", + "Network": "Syndicated search partners", + "Impressions": 729, + "Clicks": 7, + "Spend": 0.55, + "Ctr": 0.0096, + "AverageCpc": 0.08, + "ReturnOnAdSpend": 0.0, + "RevenuePerConversion": null, + "ConversionRate": null + }, + { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-08-01", + "DeviceType": "Computer", + "Network": "Bing and Yahoo! search", + "Impressions": 22, + "Clicks": 0, + "Spend": 0.0, + "Ctr": 0.0, + "AverageCpc": 0.0, + "ReturnOnAdSpend": null, + "RevenuePerConversion": null, + "ConversionRate": null + }, + { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-08-01", + "DeviceType": "Computer", + "Network": "Syndicated search partners", + "Impressions": 60, + "Clicks": 1, + "Spend": 0.11, + "Ctr": 0.0167, + "AverageCpc": 0.11, + "ReturnOnAdSpend": 0.0, + "RevenuePerConversion": null, + "ConversionRate": null + }, + { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-08-01", + "DeviceType": "Smartphone", + "Network": "Bing and Yahoo! search", + "Impressions": 1, + "Clicks": 0, + "Spend": 0.0, + "Ctr": 0.0, + "AverageCpc": 0.0, + "ReturnOnAdSpend": null, + "RevenuePerConversion": null, + "ConversionRate": null + }, + { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-08-01", + "DeviceType": "Smartphone", + "Network": "Syndicated search partners", + "Impressions": 1438, + "Clicks": 22, + "Spend": 1.62, + "Ctr": 0.015300000000000001, + "AverageCpc": 0.07, + "ReturnOnAdSpend": 0.0, + "RevenuePerConversion": null, + "ConversionRate": null + }, + { + "AccountName": "Daxtarity Inc.", + "AccountNumber": "F149GKV5", + "AccountId": 180278106, + "TimePeriod": "2021-08-01", + "DeviceType": "Tablet", + "Network": "Syndicated search partners", + "Impressions": 7, + "Clicks": 0, + "Spend": 0.0, + "Ctr": 0.0, + "AverageCpc": 0.0, + "ReturnOnAdSpend": null, + "RevenuePerConversion": null, + "ConversionRate": null + } +] diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/accounts_records.json b/airbyte-integrations/connectors/source-bing-ads/unit_tests/accounts_records.json new file mode 100644 index 0000000000000..44728c41134c8 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/accounts_records.json @@ -0,0 +1,84 @@ +[ + { + "BillToCustomerId": 251186883, + "CurrencyCode": "USD", + "AccountFinancialStatus": "ClearFinancialStatus", + "Id": 180519267, + "Language": "English", + "LastModifiedByUserId": 138225488, + "LastModifiedTime": "2021-07-09T13:16:43.337000", + "Name": "Airbyte", + "Number": "F149MJ18", + "ParentCustomerId": 251186883, + "PaymentMethodId": null, + "PaymentMethodType": null, + "PrimaryUserId": 138225488, + "AccountLifeCycleStatus": "Pending", + "TimeStamp": "AAAAAEpme9E=", + "TimeZone": "CentralTimeUSCanada", + "PauseReason": null, + "ForwardCompatibilityMap": null, + "LinkedAgencies": null, + "SalesHouseCustomerId": null, + "TaxInformation": null, + "BackUpPaymentInstrumentId": null, + "BillingThresholdAmount": null, + "BusinessAddress": { + "City": "San Francisco", + "CountryCode": "US", + "Id": 149649761, + "Line1": "350 29th Ave", + "Line2": null, + "Line3": null, + "Line4": null, + "PostalCode": "94121-1703", + "StateOrProvince": "CA", + "TimeStamp": null, + "BusinessName": "Airbyte" + }, + "AutoTagType": "Preserve", + "SoldToPaymentInstrumentId": null, + "AccountMode": "Expert" + }, + { + "BillToCustomerId": 251186883, + "CurrencyCode": "USD", + "AccountFinancialStatus": "ClearFinancialStatus", + "Id": 180278106, + "Language": "English", + "LastModifiedByUserId": 3, + "LastModifiedTime": "2021-08-23T07:06:19.147000", + "Name": "Daxtarity Inc.", + "Number": "F149GKV5", + "ParentCustomerId": 251186883, + "PaymentMethodId": 138188746, + "PaymentMethodType": "CreditCard", + "PrimaryUserId": 138225488, + "AccountLifeCycleStatus": "Active", + "TimeStamp": "AAAAAE0a41E=", + "TimeZone": "Arizona", + "PauseReason": null, + "ForwardCompatibilityMap": null, + "LinkedAgencies": null, + "SalesHouseCustomerId": null, + "TaxInformation": null, + "BackUpPaymentInstrumentId": null, + "BillingThresholdAmount": null, + "BusinessAddress": { + "City": "San Francisco", + "CountryCode": "US", + "Id": 149004358, + "Line1": "350 29th avenue", + "Line2": null, + "Line3": null, + "Line4": null, + "PostalCode": "94121", + "StateOrProvince": "CA", + "TimeStamp": null, + "BusinessName": "Daxtarity Inc." + }, + "AutoTagType": "Inactive", + "SoldToPaymentInstrumentId": null, + "AccountMode": "Expert" + } +] diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py new file mode 100644 index 0000000000000..9c5a4376f19e1 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py @@ -0,0 +1,148 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +import json +from unittest.mock import MagicMock, patch + +import pytest +import source_bing_ads +from airbyte_cdk.models import SyncMode +from source_bing_ads.client import Client +from source_bing_ads.source import AccountPerformanceReportMonthly, Accounts, AdGroups, Ads, Campaigns, SourceBingAds + + +@pytest.fixture(name="config") +def config_fixture(): + """Generates streams settings from a config file""" + CONFIG_FILE = "secrets/config.json" + with open(CONFIG_FILE, "r") as f: + return json.loads(f.read()) + + +@pytest.fixture(name="logger_mock") +def logger_mock_fixture(): + return patch("source_bing_ads.source.AirbyteLogger") + + +@patch.object(source_bing_ads.source, "Client") +def test_streams_config_based(mocked_client, config): + streams = SourceBingAds().streams(config) + assert len(streams) == 15 + + +@patch.object(source_bing_ads.source, "Client") +def test_streams_all(mocked_client): + streams = SourceBingAds().streams(MagicMock()) + assert len(streams) == 25 + + +@patch.object(source_bing_ads.source, "Client") +def test_source_check_connection_ok(mocked_client, config, logger_mock): + with patch.object(Accounts, "read_records", return_value=iter([{"Id": 180519267}, {"Id": 180278106}])): + assert SourceBingAds().check_connection(logger_mock, config=config) == (True, None) + + +@patch.object(source_bing_ads.source, "Client") +def test_source_check_connection_failed(mocked_client, config, logger_mock): + with patch.object(Accounts, "read_records", return_value=0): + assert SourceBingAds().check_connection(logger_mock, config=config)[0] is False + + +@patch.object(source_bing_ads.source, "Client") +def test_campaigns_request_params(mocked_client, config): + + campaigns = Campaigns(mocked_client, config) + + request_params = campaigns.request_params(stream_slice={"account_id": "account_id"}) + assert request_params == { + "AccountId": "account_id", + "ReturnAdditionalFields": "AdScheduleUseSearcherTimeZone BidStrategyId CpvCpmBiddingScheme DynamicDescriptionSetting DynamicFeedSetting MaxConversionValueBiddingScheme MultimediaAdsBidAdjustment TargetImpressionShareBiddingScheme TargetSetting VerifiedTrackingSetting", + } + + +@patch.object(source_bing_ads.source, "Client") +def test_campaigns_stream_slices(mocked_client, config): + + campaigns = Campaigns(mocked_client, config) + accounts_read_records = iter([{"Id": 180519267, "ParentCustomerId": 100}, {"Id": 180278106, "ParentCustomerId": 200}]) + with patch.object(Accounts, "read_records", return_value=accounts_read_records): + slices = campaigns.stream_slices() + assert list(slices) == [ + {"account_id": 180519267, "customer_id": 100}, + {"account_id": 180278106, "customer_id": 200}, + ] + + +@patch.object(source_bing_ads.source, "Client") +def test_adgroups_stream_slices(mocked_client, config): + + adgroups = AdGroups(mocked_client, config) + accounts_read_records = iter([{"Id": 180519267, "ParentCustomerId": 100}, {"Id": 180278106, "ParentCustomerId": 200}]) + campaigns_read_records = [iter([{"Id": 11}, {"Id": 22}]), iter([{"Id": 55}, {"Id": 66}])] + with patch.object(Accounts, "read_records", return_value=accounts_read_records): + with patch.object(Campaigns, "read_records", side_effect=campaigns_read_records): + slices = adgroups.stream_slices() + assert list(slices) == [ + {"campaign_id": 11, "account_id": 180519267, "customer_id": 100}, + {"campaign_id": 22, "account_id": 180519267, "customer_id": 100}, + {"campaign_id": 55, "account_id": 180278106, "customer_id": 200}, + {"campaign_id": 66, "account_id": 180278106, "customer_id": 200}, + ] + + +@patch.object(source_bing_ads.source, "Client") +def test_ads_request_params(mocked_client, config): + + ads = Ads(mocked_client, config) + + request_params = ads.request_params(stream_slice={"ad_group_id": "ad_group_id"}) + assert request_params == { + "AdGroupId": "ad_group_id", + "AdTypes": { + "AdType": ["Text", "Image", "Product", "AppInstall", "ExpandedText", "DynamicSearch", "ResponsiveAd", "ResponsiveSearch"] + }, + "ReturnAdditionalFields": "ImpressionTrackingUrls Videos LongHeadlines", + } + + +@patch.object(source_bing_ads.source, "Client") +def test_ads_stream_slices(mocked_client, config): + + ads = Ads(mocked_client, config) + + with patch.object( + AdGroups, + "stream_slices", + return_value=iter([{"account_id": 180519267, "customer_id": 100}, {"account_id": 180278106, "customer_id": 200}]), + ): + with patch.object(AdGroups, "read_records", side_effect=[iter([{"Id": 11}, {"Id": 22}]), iter([{"Id": 55}, {"Id": 66}])]): + slices = ads.stream_slices() + assert list(slices) == [ + {"ad_group_id": 11, "account_id": 180519267, "customer_id": 100}, + {"ad_group_id": 22, "account_id": 180519267, "customer_id": 100}, + {"ad_group_id": 55, "account_id": 180278106, "customer_id": 200}, + {"ad_group_id": 66, "account_id": 180278106, "customer_id": 200}, + ] + + +@patch.object(source_bing_ads.source, "Client") +def test_AccountPerformanceReportMonthly_request_params(mocked_client, config): + + accountperformancereportmonthly = AccountPerformanceReportMonthly(mocked_client, config) + request_params = accountperformancereportmonthly.request_params(account_id=180278106) + del request_params["report_request"] + assert request_params == { + "overwrite_result_file": True, + # 'report_request': , + "result_file_directory": "/tmp", + "result_file_name": "AccountPerformanceReport", + "timeout_in_milliseconds": 300000, + } + + +def test_accounts_live(config): + client = Client(**config) + accounts = Accounts(client, config) + records = accounts.read_records(SyncMode.full_refresh) + assert len(list(records)) == 4 diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java index 5d16b41cdc8d3..309aa37d5c9db 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java @@ -35,6 +35,7 @@ public OAuthImplementationFactory(final ConfigRepository configRepository, final .put("airbyte/source-instagram", new InstagramOAuthFlow(configRepository, httpClient)) .put("airbyte/source-lever-hiring", new LeverOAuthFlow(configRepository, httpClient)) .put("airbyte/source-microsoft-teams", new MicrosoftTeamsOAuthFlow(configRepository, httpClient)) + .put("airbyte/source-bing-ads", new MicrosoftBingAdsOAuthFlow(configRepository, httpClient)) .put("airbyte/source-pipedrive", new PipeDriveOAuthFlow(configRepository, httpClient)) .put("airbyte/source-quickbooks", new QuickbooksOAuthFlow(configRepository, httpClient)) .put("airbyte/source-retently", new RetentlyOAuthFlow(configRepository, httpClient)) diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/MicrosoftBingAdsOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/MicrosoftBingAdsOAuthFlow.java new file mode 100644 index 0000000000000..b355d14b92be6 --- /dev/null +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/MicrosoftBingAdsOAuthFlow.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.oauth.flows; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.oauth.BaseOAuth2Flow; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.http.HttpClient; +import java.util.Map; +import java.util.UUID; +import java.util.function.Supplier; +import org.apache.http.client.utils.URIBuilder; + +public class MicrosoftBingAdsOAuthFlow extends BaseOAuth2Flow { + + private static final String fieldName = "tenant_id"; + + public MicrosoftBingAdsOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient) { + super(configRepository, httpClient); + } + + @VisibleForTesting + public MicrosoftBingAdsOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient, final Supplier stateSupplier) { + super(configRepository, httpClient, stateSupplier); + } + + private String getScopes() { + return "offline_access%20https://ads.microsoft.com/msads.manage"; + } + + @Override + protected String formatConsentUrl(final UUID definitionId, + final String clientId, + final String redirectUrl, + final JsonNode inputOAuthConfiguration) + throws IOException { + + final String tenantId; + try { + tenantId = getConfigValueUnsafe(inputOAuthConfiguration, fieldName); + } catch (final IllegalArgumentException e) { + throw new IOException("Failed to get " + fieldName + " value from input configuration", e); + } + + try { + return new URIBuilder() + .setScheme("https") + .setHost("login.microsoftonline.com") + .setPath(tenantId + "/oauth2/v2.0/authorize") + .addParameter("client_id", clientId) + .addParameter("response_type", "code") + .addParameter("redirect_uri", redirectUrl) + .addParameter("response_mode", "query") + .addParameter("state", getState()) + .build().toString() + "&scope=" + getScopes(); + } catch (final URISyntaxException e) { + throw new IOException("Failed to format Consent URL for OAuth flow", e); + } + } + + @Override + protected Map getAccessTokenQueryParameters(final String clientId, + final String clientSecret, + final String authCode, + final String redirectUrl) { + return ImmutableMap.builder() + .put("client_id", clientId) + .put("code", authCode) + .put("redirect_uri", redirectUrl) + .put("grant_type", "authorization_code") + .build(); + } + + @Override + protected String getAccessTokenUrl(final JsonNode inputOAuthConfiguration) { + final String tenantId = getConfigValueUnsafe(inputOAuthConfiguration, fieldName); + return "https://login.microsoftonline.com/" + tenantId + "/oauth2/v2.0/token"; + } + +} diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/MicrosoftBingAdsOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/MicrosoftBingAdsOAuthFlowTest.java new file mode 100644 index 0000000000000..7a81e9790fec3 --- /dev/null +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/MicrosoftBingAdsOAuthFlowTest.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.oauth.flows; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.oauth.BaseOAuthFlow; +import java.util.Map; +import org.junit.jupiter.api.Test; + +public class MicrosoftBingAdsOAuthFlowTest extends BaseOAuthFlowTest { + + @Override + protected BaseOAuthFlow getOAuthFlow() { + return new MicrosoftBingAdsOAuthFlow(getConfigRepository(), getHttpClient(), this::getConstantState); + } + + @Override + protected String getExpectedConsentUrl() { + return "https://login.microsoftonline.com/test_tenant_id/oauth2/v2.0/authorize?client_id=test_client_id&response_type=code&redirect_uri=https%3A%2F%2Fairbyte.io&response_mode=query&state=state&scope=offline_access%20https://ads.microsoft.com/msads.manage"; + } + + @Override + protected JsonNode getInputOAuthConfiguration() { + return Jsons.jsonNode(Map.of("tenant_id", "test_tenant_id")); + } + + @Override + protected JsonNode getUserInputFromConnectorConfigSpecification() { + return getJsonSchema(Map.of("tenant_id", Map.of("type", "string"))); + } + + @Test + public void testEmptyInputCompleteDestinationOAuth() {} + + @Test + public void testDeprecatedCompleteDestinationOAuth() {} + + @Test + public void testDeprecatedCompleteSourceOAuth() {} + + @Test + public void testEmptyInputCompleteSourceOAuth() {} + +} diff --git a/docs/integrations/sources/bing-ads.md b/docs/integrations/sources/bing-ads.md index 74461c5882b49..5eb509dab2148 100644 --- a/docs/integrations/sources/bing-ads.md +++ b/docs/integrations/sources/bing-ads.md @@ -1,71 +1,104 @@ # Bing Ads -## Overview - -The Bing Ads connector syncs data from the [Bing Ads API](https://docs.microsoft.com/en-us/advertising/guides/?view=bingads-13). - -## Output schema - -This Source is capable of syncing the following resources: - -* [Accounts](https://docs.microsoft.com/en-us/advertising/customer-management-service/searchaccounts?view=bingads-13) -* [Campaigns](https://docs.microsoft.com/en-us/advertising/campaign-management-service/getcampaignsbyaccountid?view=bingads-13) -* [AdGroups](https://docs.microsoft.com/en-us/advertising/campaign-management-service/getadgroupsbycampaignid?view=bingads-13) -* [Ads](https://docs.microsoft.com/en-us/advertising/campaign-management-service/getadsbyadgroupid?view=bingads-13) - -It can also sync the following reports: - -* [AccountPerformanceReport](https://docs.microsoft.com/en-us/advertising/reporting-service/accountperformancereportrequest?view=bingads-13) -* [AdPerformanceReport](https://docs.microsoft.com/en-us/advertising/reporting-service/adperformancereportrequest?view=bingads-13) -* [AdGroupPerformanceReport](https://docs.microsoft.com/en-us/advertising/reporting-service/adgroupperformancereportrequest?view=bingads-13) -* [CampaignPerformanceReport](https://docs.microsoft.com/en-us/advertising/reporting-service/campaignperformancereportrequest?view=bingads-13) -* [BudgetSummaryReport](https://docs.microsoft.com/en-us/advertising/reporting-service/budgetsummaryreportrequest?view=bingads-13) -* [KeywordPerformanceReport](https://docs.microsoft.com/en-us/advertising/reporting-service/keywordperformancereportrequest?view=bingads-13) +This page guides you through the process of setting up the Bing Ads source connector. + +## Prerequisites (Airbyte Cloud) +* A Bing Ads account with permission to access data from accounts you want to sync + +## Prerequisites (Airbyte Open Source) +* Tenant ID +* Developer Token +* Client ID +* Client Secret +* Refresh Token +* Reports replication start date + +## Step 1: Set up Bing Ads + +1. Create a developer application using the instructions for [registering an application](https://docs.microsoft.com/en-us/advertising/guides/authentication-oauth-register?view=bingads-13) in Azure portal +2. Perform [these steps](https://docs.microsoft.com/en-us/advertising/guides/authentication-oauth-consent?view=bingads-13l) to get auth code, and use that to [get a refresh token](https://docs.microsoft.com/en-us/advertising/guides/authentication-oauth-get-tokens?view=bingads-13). For reference, the full authentication process described [here](https://docs.microsoft.com/en-us/advertising/guides/get-started?view=bingads-13#access-token). Be aware that the refresh token will expire in 90 days. You need to repeat the auth process to get a new refresh token. +3. Find your Microsoft developer token by following [these instructions](https://docs.microsoft.com/en-us/advertising/guides/get-started?view=bingads-13#get-developer-token) +4. Optionally, if your oauth app lives under a custom tenant which cannot use Microsoft's recommended `common` tenant, make sure to get the tenant ID ready for input when configuring the connector. The tenant will be used in the auth URL e.g: `https://login.microsoftonline.com//oauth2/v2.0/authorize`. + +## Step 2: Set up the source connector in Airbyte + +**For Airbyte Cloud:** + +1. [Log into your Airbyte Cloud](https://cloud.airbyte.io/workspaces) account. +2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**. +3. On the source setup page, select **Bing Ads** from the Source type dropdown and enter a name for this connector. +4. Add Tenant ID +5. Click `Authenticate your account`. +6. Log in and Authorize to the BingAds account +7. Choose required Start date and type of aggregation report +8. click `Set up source`. + +**For Airbyte OSS:** + +1. Go to local Airbyte page. +2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**. +3. On the Set up the source page, enter the name for the connector and select **Bing Ads** from the Source type dropdown. +4. Add Tenant ID +5. Copy and paste info from step 1: + * client ID + * client secret + * refresh_token + * A developer token +7. Choose required Start date and type of aggregation report +8. Click `Set up source`. + +## Supported sync modes + +The Bing Ads source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): + - Full Refresh + - Incremental + +## Supported Streams +Basic streams: +* [accounts](https://docs.microsoft.com/en-us/advertising/customer-management-service/searchaccounts?view=bingads-13) +* [campaigns](https://docs.microsoft.com/en-us/advertising/campaign-management-service/getcampaignsbyaccountid?view=bingads-13) +* [ad_groups](https://docs.microsoft.com/en-us/advertising/campaign-management-service/getadgroupsbycampaignid?view=bingads-13) +* [ads](https://docs.microsoft.com/en-us/advertising/campaign-management-service/getadsbyadgroupid?view=bingads-13) + +Report Streams: +* [budget_summary_report](https://docs.microsoft.com/en-us/advertising/reporting-service/budgetsummaryreportrequest?view=bingads-13) +* [account_performance_report_hourly](https://docs.microsoft.com/en-us/advertising/reporting-service/accountperformancereportrequest?view=bingads-13) +* [account_performance_report_daily](https://docs.microsoft.com/en-us/advertising/reporting-service/accountperformancereportrequest?view=bingads-13) +* [account_performance_report_weekly](https://docs.microsoft.com/en-us/advertising/reporting-service/accountperformancereportrequest?view=bingads-13) +* [account_performance_report_monthly](https://docs.microsoft.com/en-us/advertising/reporting-service/accountperformancereportrequest?view=bingads-13) +* [ad_group_performance_report_hourly](https://docs.microsoft.com/en-us/advertising/reporting-service/adgroupperformancereportrequest?view=bingads-13) +* [ad_group_performance_report_daily](https://docs.microsoft.com/en-us/advertising/reporting-service/adgroupperformancereportrequest?view=bingads-13) +* [ad_group_performance_report_weekly](https://docs.microsoft.com/en-us/advertising/reporting-service/adgroupperformancereportrequest?view=bingads-13) +* [ad_group_performance_report_monthly](https://docs.microsoft.com/en-us/advertising/reporting-service/adgroupperformancereportrequest?view=bingads-13) +* [ad_performance_report_hourly](https://docs.microsoft.com/en-us/advertising/reporting-service/adperformancereportrequest?view=bingads-13) +* [ad_performance_report_daily](https://docs.microsoft.com/en-us/advertising/reporting-service/adperformancereportrequest?view=bingads-13) +* [ad_performance_report_weekly](https://docs.microsoft.com/en-us/advertising/reporting-service/adperformancereportrequest?view=bingads-13) +* [ad_performance_report_monthly](https://docs.microsoft.com/en-us/advertising/reporting-service/adperformancereportrequest?view=bingads-13) +* [campaign_performance_report_hourly](https://docs.microsoft.com/en-us/advertising/reporting-service/campaignperformancereportrequest?view=bingads-13) +* [campaign_performance_report_daily](https://docs.microsoft.com/en-us/advertising/reporting-service/campaignperformancereportrequest?view=bingads-13) +* [campaign_performance_report_weekly](https://docs.microsoft.com/en-us/advertising/reporting-service/campaignperformancereportrequest?view=bingads-13) +* [campaign_performance_report_monthly](https://docs.microsoft.com/en-us/advertising/reporting-service/campaignperformancereportrequest?view=bingads-13) +* [keyword_performance_report_hourly](https://docs.microsoft.com/en-us/advertising/reporting-service/keywordperformancereportrequest?view=bingads-13) +* [keyword_performance_report_daily](https://docs.microsoft.com/en-us/advertising/reporting-service/keywordperformancereportrequest?view=bingads-13) +* [keyword_performance_report_weekly](https://docs.microsoft.com/en-us/advertising/reporting-service/keywordperformancereportrequest?view=bingads-13) +* [keyword_performance_report_monthly](https://docs.microsoft.com/en-us/advertising/reporting-service/keywordperformancereportrequest?view=bingads-13) + +For more information, see the [Bing Ads API](https://docs.microsoft.com/en-us/advertising/guides/?view=bingads-13). ### Report Aggregation All reports synced by this connector can be aggregated using hourly, daily, weekly, or monthly windows. Performance data is aggregated using the selected window. For example, if you select the daily-aggregation flavor of a report, the report will contain a row for each day for the duration of the report. Each row will indicate the number of impressions recorded on that day. -A report's aggregation window is indicated in its name e.g: `account_performance_report_hourly` is the Account Performance Reported aggregated using an hourly window. - -### Features - -| Feature | Supported?\(Yes/No\) | Notes | -| :--- |:---------------------| :--- | -| Full Refresh Sync | Yes | | -| Incremental Sync | Yes | | -| Namespaces | No | | +A report's aggregation window is indicated in its name e.g: `account_performance_report_hourly` is the Account Performance Reported aggregated using an hourly window. ### Performance considerations API limits number of requests for all Microsoft Advertising clients. You can find detailied info [here](https://docs.microsoft.com/en-us/advertising/guides/services-protocol?view=bingads-13#throttling) -## Getting started (Airbyte Open Source) -### Requirements -* A Microsoft User account with access to at least one Microsoft Advertising account -* A Microsoft Ads Customer ID -* Your Microsoft User ID -* A developer application with access to: - * client ID - * client secret - * A developer token - * Optionally, a tenant ID -* A refresh token generated using the above developer application credentials -* (Optional) Ad Account IDs you want to access, if you want to limit replication to specific ad accounts - -### Setup Guide -* Create a developer application using the instructions for [registering an application](https://docs.microsoft.com/en-us/advertising/guides/authentication-oauth-register?view=bingads-13) in Azure portal -* Perform [these steps](https://docs.microsoft.com/en-us/advertising/guides/authentication-oauth-consent?view=bingads-13l) to get auth code, and use that to [get a refresh token](https://docs.microsoft.com/en-us/advertising/guides/authentication-oauth-get-tokens?view=bingads-13). For reference, the full authentication process described [here](https://docs.microsoft.com/en-us/advertising/guides/get-started?view=bingads-13#access-token). Be aware that the refresh token will expire in 90 days. You need to repeat the auth process to get a new refresh token. -* Find your Microsoft developer token by following [these instructions](https://docs.microsoft.com/en-us/advertising/guides/get-started?view=bingads-13#get-developer-token) -* Find your customer ID and User ID by visiting the following URL: https://ui.ads.microsoft.com/campaign/Campaigns.m then copying the CID & UID parameters from the URL in the address bar. For example, once you visit the URL above, you'll notice it will have changed to an address of the form https://ui.ads.microsoft.com/campaign/vnext/overview?uid=USER_ID&cid=CUSTOMER_ID&aid=180534868 -- the customer ID is the value in the part of the URL that looks like `cid=THIS_IS_THE_CUSTOMER_ID&`, and the user ID is the value in front of `uid` e.g: `uid=THIS_IS_THE_USER_ID&`. -* Optionally, if you want to replicate data from specific ad account IDs (you can configure the Bing Ads connector to replicate data from all accounts you have access to, or only from some), then also grab the account IDs you want by visiting the [Accounts Summary](https://ui.ads.microsoft.com/campaign/vnext/accounts/performance) page, clicking on each of the accounts you want under the `Account name` column, then repeating the process described earlier to get the `aid` parameter in the URL that looks like `aid=ACCOUNT_ID&`. You'll need to do this process once for each account from which you want to replicate data. -* Optionally, if your oauth app lives under a custom tenant which cannot use Microsoft's recommended `common` tenant, make sure to get the tenant ID ready for input when configuring the connector. The tenant will be used in the auth URL e.g: `https://login.microsoftonline.com//oauth2/v2.0/authorize`. - - - ## Changelog | Version | Date | Pull Request | Subject | |:--------| :--- |:---------------------------------------------------------| :--- | +| 0.1.7 | 2022-05-17 | [12937](https://github.com/airbytehq/airbyte/pull/12937) | Added OAuth2.0 authentication method, removed `redirect_uri` from input configuration | 0.1.6 | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500) | Improve input configuration copy | | 0.1.5 | 2022-01-01 | [11652](https://github.com/airbytehq/airbyte/pull/11652) | Rebump attempt after DockerHub failure at registring the 0.1.4 | | 0.1.4 | 2022-03-22 | [11311](https://github.com/airbytehq/airbyte/pull/11311) | Added optional Redirect URI & Tenant ID to spec |