Source Google Ads to GA: update spec and docs (#12807)

* #12650: source google ads to GA: mock input config for unit tests * #12650 source google ads to GA: doc updates * #12650 source google ads to GA: upd changelog * #12650 source GAds to GA: doc review fixes * auto-bump connector version Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
airbytehq · May 13, 2022 · e49faed · e49faed
1 parent 25115ec
commit e49faed
Show file tree

Hide file tree

Showing 7 changed files with 155 additions and 121 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -311,7 +311,7 @@
 - name: Google Ads
   sourceDefinitionId: 253487c0-2246-43ba-a21f-5116b20a2c50
   dockerRepository: airbyte/source-google-ads
-  dockerImageTag: 0.1.37
+  dockerImageTag: 0.1.38
   documentationUrl: https://docs.airbyte.io/integrations/sources/google-ads
   icon: google-adwords.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -2688,7 +2688,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-google-ads:0.1.37"
+- dockerImage: "airbyte/source-google-ads:0.1.38"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/google-ads"
     connectionSpecification:
@@ -2715,34 +2715,39 @@
             developer_token:
               type: "string"
               title: "Developer Token"
+              order: 0
               description: "Developer token granted by Google to use their APIs. More\
                 \ instruction on how to find this value in our <a href=\"https://docs.airbyte.com/integrations/sources/google-ads#setup-guide\"\
                 >docs</a>"
               airbyte_secret: true
             client_id:
               type: "string"
               title: "Client ID"
+              order: 1
               description: "The Client ID of your Google Ads developer application.\
                 \ More instruction on how to find this value in our <a href=\"https://docs.airbyte.com/integrations/sources/google-ads#setup-guide\"\
                 >docs</a>"
             client_secret:
               type: "string"
               title: "Client Secret"
+              order: 2
               description: "The Client Secret of your Google Ads developer application.\
                 \ More instruction on how to find this value in our <a href=\"https://docs.airbyte.com/integrations/sources/google-ads#setup-guide\"\
                 >docs</a>"
               airbyte_secret: true
-            access_token:
+            refresh_token:
               type: "string"
-              title: "Access Token"
-              description: "Access Token for making authenticated requests. More instruction\
+              title: "Refresh Token"
+              order: 3
+              description: "The token for obtaining a new access token. More instruction\
                 \ on how to find this value in our <a href=\"https://docs.airbyte.com/integrations/sources/google-ads#setup-guide\"\
                 >docs</a>"
               airbyte_secret: true
-            refresh_token:
+            access_token:
               type: "string"
-              title: "Refresh Token"
-              description: "The token for obtaining a new access token. More instruction\
+              title: "Access Token (Optional)"
+              order: 4
+              description: "Access Token for making authenticated requests. More instruction\
                 \ on how to find this value in our <a href=\"https://docs.airbyte.com/integrations/sources/google-ads#setup-guide\"\
                 >docs</a>"
               airbyte_secret: true

diff --git a/airbyte-integrations/connectors/source-google-ads/Dockerfile b/airbyte-integrations/connectors/source-google-ads/Dockerfile
@@ -13,5 +13,5 @@ COPY main.py ./
 
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.37
+LABEL io.airbyte.version=0.1.38
 LABEL io.airbyte.name=airbyte/source-google-ads
diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json
@@ -22,31 +22,36 @@
           "developer_token": {
             "type": "string",
             "title": "Developer Token",
+            "order": 0,
             "description": "Developer token granted by Google to use their APIs. More instruction on how to find this value in our <a href=\"https://docs.airbyte.com/integrations/sources/google-ads#setup-guide\">docs</a>",
             "airbyte_secret": true
           },
           "client_id": {
             "type": "string",
             "title": "Client ID",
+            "order": 1,
             "description": "The Client ID of your Google Ads developer application. More instruction on how to find this value in our <a href=\"https://docs.airbyte.com/integrations/sources/google-ads#setup-guide\">docs</a>"
           },
           "client_secret": {
             "type": "string",
             "title": "Client Secret",
+            "order": 2,
             "description": "The Client Secret of your Google Ads developer application. More instruction on how to find this value in our <a href=\"https://docs.airbyte.com/integrations/sources/google-ads#setup-guide\">docs</a>",
             "airbyte_secret": true
           },
-          "access_token": {
-            "type": "string",
-            "title": "Access Token",
-            "description": "Access Token for making authenticated requests. More instruction on how to find this value in our <a href=\"https://docs.airbyte.com/integrations/sources/google-ads#setup-guide\">docs</a>",
-            "airbyte_secret": true
-          },
           "refresh_token": {
             "type": "string",
             "title": "Refresh Token",
+            "order": 3,
             "description": "The token for obtaining a new access token. More instruction on how to find this value in our <a href=\"https://docs.airbyte.com/integrations/sources/google-ads#setup-guide\">docs</a>",
             "airbyte_secret": true
+          },
+          "access_token": {
+            "type": "string",
+            "title": "Access Token (Optional)",
+            "order": 4,
+            "description": "Access Token for making authenticated requests. More instruction on how to find this value in our <a href=\"https://docs.airbyte.com/integrations/sources/google-ads#setup-guide\">docs</a>",
+            "airbyte_secret": true
           }
         }
       },

diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/conftest.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/conftest.py
@@ -2,15 +2,43 @@
 # Copyright (c) 2021 Airbyte, Inc., all rights reserved.
 #
 
-import json
-
 import pytest
 
 
 @pytest.fixture(scope="session", name="config")
-def config_fixture():
-    with open("secrets/config.json", "r") as config_file:
-        return json.load(config_file)
+def test_config():
+    config = {
+        "credentials": {
+            "developer_token": "test_token",
+            "client_id": "test_client_id",
+            "client_secret": "test_client_secret",
+            "refresh_token": "test_refresh_token",
+        },
+        "customer_id": "123",
+        "start_date": "2021-01-01",
+        "conversion_window_days": 14,
+        "custom_queries": [
+            {
+                "query": "SELECT campaign.accessible_bidding_strategy, segments.ad_destination_type, campaign.start_date, campaign.end_date FROM campaign",
+                "primary_key": None,
+                "cursor_field": "campaign.start_date",
+                "table_name": "happytable",
+            },
+            {
+                "query": "SELECT segments.ad_destination_type, segments.ad_network_type, segments.day_of_week, customer.auto_tagging_enabled, customer.id, metrics.conversions, campaign.start_date FROM campaign",
+                "primary_key": "customer.id",
+                "cursor_field": None,
+                "table_name": "unhappytable",
+            },
+            {
+                "query": "SELECT ad_group.targeting_setting.target_restrictions FROM ad_group",
+                "primary_key": "customer.id",
+                "cursor_field": None,
+                "table_name": "ad_group_custom",
+            },
+        ],
+    }
+    return config
 
 
 @pytest.fixture(autouse=True)

diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py
@@ -16,26 +16,10 @@
 from .common import MockGoogleAdsClient as MockGoogleAdsClient
 
 
-@pytest.fixture(scope="module")
-def test_config():
-    config = {
-        "credentials": {
-            "developer_token": "test_token",
-            "client_id": "test_client_id",
-            "client_secret": "test_client_secret",
-            "refresh_token": "test_refresh_token",
-        },
-        "customer_id": "123",
-        "start_date": "2021-01-01",
-        "conversion_window_days": 14,
-    }
-    return config
-
-
 @pytest.fixture
-def mock_ads_client(mocker):
+def mock_ads_client(mocker, config):
     """Mock google ads library method, so it returns mocked Client"""
-    mocker.patch("source_google_ads.google_ads.GoogleAdsClient.load_from_dict", return_value=MockGoogleAdsClient(test_config))
+    mocker.patch("source_google_ads.google_ads.GoogleAdsClient.load_from_dict", return_value=MockGoogleAdsClient(config))
 
 
 # EXPIRED_PAGE_TOKEN exception will be raised when page token has expired.
@@ -81,7 +65,7 @@ def send_request(self, query: str, customer_id: str):
             return mock_response_2()
 
 
-def test_page_token_expired_retry_succeeds(mock_ads_client, test_config):
+def test_page_token_expired_retry_succeeds(mock_ads_client, config):
     """
     Page token expired while reading records on date 2021-01-03
     The latest read record is {"segments.date": "2021-01-03", "click_view.gclid": "4"}
@@ -90,11 +74,11 @@ def test_page_token_expired_retry_succeeds(mock_ads_client, test_config):
     """
     stream_slice = {"start_date": "2021-01-01", "end_date": "2021-01-15"}
 
-    google_api = MockGoogleAds(credentials=test_config["credentials"], customer_id=test_config["customer_id"])
+    google_api = MockGoogleAds(credentials=config["credentials"], customer_id=config["customer_id"])
     incremental_stream_config = dict(
         api=google_api,
-        conversion_window_days=test_config["conversion_window_days"],
-        start_date=test_config["start_date"],
+        conversion_window_days=config["conversion_window_days"],
+        start_date=config["start_date"],
         time_zone="local",
         end_date="2021-04-04",
     )
@@ -139,18 +123,18 @@ def send_request(self, query: str, customer_id: str):
             return mock_response_fails_2()
 
 
-def test_page_token_expired_retry_fails(mock_ads_client, test_config):
+def test_page_token_expired_retry_fails(mock_ads_client, config):
     """
     Page token has expired while reading records within date "2021-01-03", it should raise error,
     because Google Ads API doesn't allow filter by datetime.
     """
     stream_slice = {"start_date": "2021-01-01", "end_date": "2021-01-15"}
 
-    google_api = MockGoogleAdsFails(credentials=test_config["credentials"], customer_id=test_config["customer_id"])
+    google_api = MockGoogleAdsFails(credentials=config["credentials"], customer_id=config["customer_id"])
     incremental_stream_config = dict(
         api=google_api,
-        conversion_window_days=test_config["conversion_window_days"],
-        start_date=test_config["start_date"],
+        conversion_window_days=config["conversion_window_days"],
+        start_date=config["start_date"],
         time_zone="local",
         end_date="2021-04-04",
     )
@@ -181,19 +165,19 @@ def send_request(self, query: str, customer_id: str):
         return mock_response_fails_one_date()
 
 
-def test_page_token_expired_it_should_fail_date_range_1_day(mock_ads_client, test_config):
+def test_page_token_expired_it_should_fail_date_range_1_day(mock_ads_client, config):
     """
     Page token has expired while reading records within date "2021-01-03",
     it should raise error, because Google Ads API doesn't allow filter by datetime.
     Minimum date range is 1 day.
     """
     stream_slice = {"start_date": "2021-01-03", "end_date": "2021-01-04"}
 
-    google_api = MockGoogleAdsFailsOneDate(credentials=test_config["credentials"], customer_id=test_config["customer_id"])
+    google_api = MockGoogleAdsFailsOneDate(credentials=config["credentials"], customer_id=config["customer_id"])
     incremental_stream_config = dict(
         api=google_api,
-        conversion_window_days=test_config["conversion_window_days"],
-        start_date=test_config["start_date"],
+        conversion_window_days=config["conversion_window_days"],
+        start_date=config["start_date"],
         time_zone="local",
         end_date="2021-04-04",
     )