🎉 Source Bing Ads: implement OAuth2.0 support, remove `redirect_uri…

…`, change `Account ID` to `User ID` in spec (#12937) * implemented oauth support * bumped version * added changelog * changed releaseStage to beta * removed redirect_uri from spec * removed user_id, customer_id from spec * fixed invalid_config * fixed flakeCheck * fixed broken schema for account stream * added old config support * added new unit test to increase test coverage to 91% * removed unneccessary tests * formated code * updated after review * added oauth implementation java test * updated spec * updated after review * updated docs * auto-bump connector version Co-authored-by: Vadym Ratniuk <midavadim@yahoo.com> Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
airbytehq · May 23, 2022 · e56922f · e56922f
1 parent 643ddfc
commit e56922f
Show file tree

Hide file tree

Showing 19 changed files with 1,027 additions and 286 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -104,11 +104,11 @@
 - name: Bing Ads
   sourceDefinitionId: 47f25999-dd5e-4636-8c39-e7cea2453331
   dockerRepository: airbyte/source-bing-ads
-  dockerImageTag: 0.1.6
+  dockerImageTag: 0.1.7
   documentationUrl: https://docs.airbyte.io/integrations/sources/bing-ads
   icon: bingads.svg
   sourceType: api
-  releaseStage: alpha
+  releaseStage: beta
 - name: Braintree
   sourceDefinitionId: 63cea06f-1c75-458d-88fe-ad48c7cb27fd
   dockerRepository: airbyte/source-braintree

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -801,115 +801,63 @@
     - "overwrite"
     - "append"
     - "append_dedup"
-- dockerImage: "airbyte/source-bing-ads:0.1.6"
+- dockerImage: "airbyte/source-bing-ads:0.1.7"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/bing-ads"
     connectionSpecification:
       $schema: "http://json-schema.org/draft-07/schema#"
       title: "Bing Ads Spec"
       type: "object"
       required:
-      - "accounts"
-      - "client_id"
-      - "client_secret"
-      - "customer_id"
       - "developer_token"
+      - "client_id"
       - "refresh_token"
-      - "user_id"
       - "reports_start_date"
       - "hourly_reports"
       - "daily_reports"
       - "weekly_reports"
       - "monthly_reports"
-      additionalProperties: false
+      additionalProperties: true
       properties:
-        accounts:
-          title: "Accounts to replicate data from"
-          type: "object"
-          description: ""
-          oneOf:
-          - title: "All Accounts"
-            additionalProperties: false
-            description: "Replicate data from all accounts to which you have access."
-            required:
-            - "selection_strategy"
-            properties:
-              selection_strategy:
-                type: "string"
-                const: "all"
-          - title: "Specific Accounts"
-            additionalProperties: false
-            description: "Fetch data for subset of account IDs."
-            required:
-            - "ids"
-            - "selection_strategy"
-            properties:
-              selection_strategy:
-                type: "string"
-                const: "subset"
-              ids:
-                type: "array"
-                title: "Account IDs"
-                description: "List of the account IDs from which data will be replicated."
-                items:
-                  type: "string"
-                minItems: 1
-                uniqueItems: true
+        auth_method:
+          type: "string"
+          const: "oauth2.0"
+        tenant_id:
+          type: "string"
+          title: "Tenant ID"
+          description: "The Tenant ID of your Microsoft Advertising developer application.\
+            \ Set this to \"common\" unless you know you need a different value."
+          airbyte_secret: true
+          default: "common"
+          order: 0
         client_id:
           type: "string"
           title: "Client ID"
           description: "The Client ID of your Microsoft Advertising developer application."
           airbyte_secret: true
-          order: 0
+          order: 1
         client_secret:
           type: "string"
           title: "Client Secret"
           description: "The Client Secret of your Microsoft Advertising developer\
             \ application."
+          default: ""
           airbyte_secret: true
-          order: 1
+          order: 2
         refresh_token:
           type: "string"
           title: "Refresh Token"
           description: "Refresh Token to renew the expired Access Token."
           airbyte_secret: true
-          order: 2
+          order: 3
         developer_token:
           type: "string"
           title: "Developer Token"
-          description: "Developer token associated with user."
+          description: "Developer token associated with user. See more info <a href=\"\
+            https://docs.microsoft.com/en-us/advertising/guides/get-started?view=bingads-13#get-developer-token\"\
+            > in the docs</a>."
           airbyte_secret: true
-          order: 3
-        tenant_id:
-          type: "string"
-          title: "Tenant ID"
-          description: "The Tenant ID of your Microsoft Advertising developer application.\
-            \ Set this to \"common\" unless you know you need a different value."
-          airbyte_secret: true
-          default: "common"
           order: 4
-        redirect_uri:
-          type: "string"
-          title: "Redirect URI (Optional)"
-          description: "The Redirect URI of your Microsoft Advertising developer application.\
-            \ Leave this empty unless you know that you need it."
-          airbyte_secret: true
-          default: ""
-          order: 5
-        customer_id:
-          type: "string"
-          title: "Customer ID"
-          description: "Your Bing Customer ID. See the \"Getting Started\" section\
-            \ in <a href=\"https://docs.airbyte.com/integrations/sources/bing-ads\"\
-            >the docs</a> for information on how to obtain this ID"
-          order: 6
-        user_id:
-          type: "string"
-          title: "Account ID"
-          description: "Bing Ads Account ID. See the \"Getting Started\" section in\
-            \ <a href=\"https://docs.airbyte.com/integrations/sources/bing-ads\">the\
-            \ docs</a> for information on how to obtain this ID"
-          order: 7
         reports_start_date:
           type: "string"
           title: "Reports replication start date"
@@ -918,7 +866,7 @@
           description: "The start date from which to begin replicating report data.\
             \ Any data generated before this date will not be replicated in reports.\
             \ This is a UTC date in YYYY-MM-DD format."
-          order: 8
+          order: 5
         hourly_reports:
           title: "Enable hourly-aggregate reports"
           type: "boolean"
@@ -954,6 +902,48 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
+    advanced_auth:
+      auth_flow_type: "oauth2.0"
+      predicate_key:
+      - "auth_method"
+      predicate_value: "oauth2.0"
+      oauth_config_specification:
+        oauth_user_input_from_connector_config_specification:
+          type: "object"
+          additionalProperties: false
+          properties:
+            tenant_id:
+              type: "string"
+              path_in_connector_config:
+              - "tenant_id"
+        complete_oauth_output_specification:
+          type: "object"
+          additionalProperties: false
+          properties:
+            refresh_token:
+              type: "string"
+              path_in_connector_config:
+              - "refresh_token"
+        complete_oauth_server_input_specification:
+          type: "object"
+          additionalProperties: false
+          properties:
+            client_id:
+              type: "string"
+            client_secret:
+              type: "string"
+        complete_oauth_server_output_specification:
+          type: "object"
+          additionalProperties: false
+          properties:
+            client_id:
+              type: "string"
+              path_in_connector_config:
+              - "client_id"
+            client_secret:
+              type: "string"
+              path_in_connector_config:
+              - "client_secret"
 - dockerImage: "airbyte/source-braintree:0.1.3"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/braintree"

diff --git a/airbyte-integrations/connectors/source-bing-ads/Dockerfile b/airbyte-integrations/connectors/source-bing-ads/Dockerfile
@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.6
+LABEL io.airbyte.version=0.1.7
 LABEL io.airbyte.name=airbyte/source-bing-ads
diff --git a/airbyte-integrations/connectors/source-bing-ads/README.md b/airbyte-integrations/connectors/source-bing-ads/README.md
@@ -116,7 +116,8 @@ Customize `acceptance-test-config.yml` file to configure tests. See [Source Acce
 If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
 To run your integration tests with acceptance tests, from the connector root, run
 ```
-python -m pytest integration_tests -p integration_tests.acceptance
+docker build . --no-cache -t airbyte/source-bing-ads:dev \
+&& python -m pytest -p source_acceptance_test.plugin
 ```
 To run your integration tests with docker
 

diff --git a/airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml
@@ -3,6 +3,8 @@ tests:
   spec:
     - spec_path: "source_bing_ads/spec.json"
   connection:
+    - config_path: "secrets/config_old.json"
+      status: "succeed"
     - config_path: "secrets/config.json"
       status: "succeed"
     - config_path: "integration_tests/invalid_config.json"

diff --git a/airbyte-integrations/connectors/source-bing-ads/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-bing-ads/integration_tests/invalid_config.json
@@ -1,16 +1,12 @@
 {
-  "accounts": { "selection_strategy": "all" },
-  "user_id": "2222",
-  "customer_id": "1111",
-  "developer_token": "asgag4gwag3",
   "refresh_token": "as2Ggas23gsa236gasgaskjfhas7i8ygf78as7osa7gy87asg8as7tg6as",
-  "client_secret": "1234",
+  "client_secret": "",
   "client_id": "123",
-  "tenant_id": "common",
-  "redirect_uri": "",
+  "developer_token": "asgag4gwag3",
   "reports_start_date": "2018-11-13",
-  "hourly_reports": true,
+  "hourly_reports": false,
   "daily_reports": false,
-  "weekly_reports": false,
-  "monthly_reports": true
+  "weekly_reports": true,
+  "monthly_reports": true,
+  "tenant_id": "common"
 }
diff --git a/airbyte-integrations/connectors/source-bing-ads/setup.py b/airbyte-integrations/connectors/source-bing-ads/setup.py
@@ -5,7 +5,7 @@
 
 from setuptools import find_packages, setup
 
-MAIN_REQUIREMENTS = ["airbyte-cdk", "bingads~=13.0.11", "vcrpy==4.1.1", "backoff==1.10.0", "pendulum==2.1.2"]
+MAIN_REQUIREMENTS = ["airbyte-cdk", "bingads~=13.0.13", "vcrpy==4.1.1", "backoff==1.10.0", "pendulum==2.1.2"]
 
 TEST_REQUIREMENTS = [
     "pytest~=6.1",

diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py
@@ -36,44 +36,52 @@ class Client:
 
     def __init__(
         self,
-        developer_token: str,
-        customer_id: str,
-        client_secret: str,
-        client_id: str,
         tenant_id: str,
-        redirect_uri: str,
-        refresh_token: str,
         reports_start_date: str,
         hourly_reports: bool,
         daily_reports: bool,
         weekly_reports: bool,
         monthly_reports: bool,
+        developer_token: str = None,
+        client_id: str = None,
+        client_secret: str = None,
+        refresh_token: str = None,
         **kwargs: Mapping[str, Any],
     ) -> None:
         self.authorization_data: Mapping[str, AuthorizationData] = {}
-        self.authentication = OAuthWebAuthCodeGrant(
-            client_id,
-            client_secret,
-            redirect_uri,
-            tenant=tenant_id,
-        )
-
         self.refresh_token = refresh_token
-        self.customer_id = customer_id
         self.developer_token = developer_token
         self.hourly_reports = hourly_reports
         self.daily_reports = daily_reports
         self.weekly_reports = weekly_reports
         self.monthly_reports = monthly_reports
 
+        self.client_id = client_id
+        self.client_secret = client_secret
+
+        self.authentication = self._get_auth_client(client_id, tenant_id, client_secret)
         self.oauth: OAuthTokens = self._get_access_token()
         self.reports_start_date = pendulum.parse(reports_start_date).astimezone(tz=timezone.utc)
 
+    def _get_auth_client(self, client_id: str, tenant_id: str, client_secret: str = None) -> OAuthWebAuthCodeGrant:
+        # https://github.com/BingAds/BingAds-Python-SDK/blob/e7b5a618e87a43d0a5e2c79d9aa4626e208797bd/bingads/authorization.py#L390
+        auth_creds = {
+            "client_id": client_id,
+            "redirection_uri": "",  # should be empty string
+            "client_secret": None,
+            "tenant": tenant_id,
+        }
+        # the `client_secret` should be provided for `non-public clients` only
+        # https://docs.microsoft.com/en-us/advertising/guides/authentication-oauth-get-tokens?view=bingads-13#request-accesstoken
+        if client_secret and client_secret != "":
+            auth_creds["client_secret"] = client_secret
+        return OAuthWebAuthCodeGrant(**auth_creds)
+
     @lru_cache(maxsize=None)
-    def _get_auth_data(self, account_id: Optional[str] = None) -> AuthorizationData:
+    def _get_auth_data(self, customer_id: str = None, account_id: Optional[str] = None) -> AuthorizationData:
         return AuthorizationData(
             account_id=account_id,
-            customer_id=self.customer_id,
+            customer_id=customer_id,
             developer_token=self.developer_token,
             authentication=self.authentication,
         )
@@ -124,6 +132,7 @@ def _request(
         self,
         service_name: Optional[str],
         operation_name: str,
+        customer_id: Optional[str],
         account_id: Optional[str],
         params: Mapping[str, Any],
         is_report_service: bool = False,
@@ -135,32 +144,34 @@ def _request(
             self.oauth = self._get_access_token()
 
         if is_report_service:
-            service = self._get_reporting_service(account_id=account_id)
+            service = self._get_reporting_service(customer_id=customer_id, account_id=account_id)
         else:
-            service = self.get_service(service_name=service_name, account_id=account_id)
+            service = self.get_service(service_name=service_name, customer_id=customer_id, account_id=account_id)
 
         return getattr(service, operation_name)(**params)
 
     @lru_cache(maxsize=None)
     def get_service(
         self,
         service_name: str,
+        customer_id: str = None,
         account_id: Optional[str] = None,
     ) -> ServiceClient:
         return ServiceClient(
             service=service_name,
             version=self.api_version,
-            authorization_data=self._get_auth_data(account_id),
+            authorization_data=self._get_auth_data(customer_id, account_id),
             environment=self.environment,
         )
 
     @lru_cache(maxsize=None)
     def _get_reporting_service(
         self,
+        customer_id: Optional[str] = None,
         account_id: Optional[str] = None,
     ) -> ServiceClient:
         return ReportingServiceManager(
-            authorization_data=self._get_auth_data(account_id),
+            authorization_data=self._get_auth_data(customer_id, account_id),
             poll_interval_in_milliseconds=self.report_poll_interval,
             environment=self.environment,
         )