Source Hubspot: remove API Key authentication type (#18546)

* Source Hubspot: remove API Key authentication type * Source Hubspot: update changelog * Source Hubspot: update sample files * Source Hubspot: update integration test * Update hubspot.md Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
airbytehq · Dec 9, 2022 · fd13d43 · fd13d43
1 parent 80e5f29
commit fd13d43
Show file tree

Hide file tree

Showing 19 changed files with 102 additions and 251 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -708,7 +708,7 @@
 - name: HubSpot
   sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c
   dockerRepository: airbyte/source-hubspot
-  dockerImageTag: 0.2.3
+  dockerImageTag: 0.3.0
   documentationUrl: https://docs.airbyte.com/integrations/sources/hubspot
   icon: hubspot.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -6203,7 +6203,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-hubspot:0.2.3"
+- dockerImage: "airbyte/source-hubspot:0.3.0"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/hubspot"
     connectionSpecification:
@@ -6269,25 +6269,7 @@
                 - "refresh_token"
                 airbyte_secret: true
           - type: "object"
-            title: "API key"
-            required:
-            - "api_key"
-            - "credentials_title"
-            properties:
-              credentials_title:
-                type: "string"
-                title: "Credentials"
-                description: "Name of the credentials set"
-                const: "API Key Credentials"
-                order: 0
-              api_key:
-                title: "API key"
-                description: "HubSpot API Key. See the <a href=\"https://knowledge.hubspot.com/integrations/how-do-i-get-my-hubspot-api-key\"\
-                  >Hubspot docs</a> if you need help finding this key."
-                type: "string"
-                airbyte_secret: true
-          - type: "object"
-            title: "Private APP"
+            title: "Private App"
             required:
             - "access_token"
             - "credentials_title"

diff --git a/airbyte-integrations/connectors/source-hubspot/Dockerfile b/airbyte-integrations/connectors/source-hubspot/Dockerfile
@@ -34,5 +34,5 @@ COPY source_hubspot ./source_hubspot
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.3
+LABEL io.airbyte.version=0.3.0
 LABEL io.airbyte.name=airbyte/source-hubspot
diff --git a/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml b/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml
@@ -1,107 +1,91 @@
 connector_image: airbyte/source-hubspot:dev
-tests:
+acceptance_tests:
   spec:
-    - spec_path: "source_hubspot/spec.yaml"
+    tests:
+      - spec_path: "source_hubspot/spec.yaml"
+        backward_compatibility_tests_config:
+          # API Key authentication is deleted from authorization choices so it is not backward compatible with previous spec
+          disable_for_version: "0.2.3"
   connection:
-    - config_path: "secrets/config.json"
-      status: "succeed"
-    - config_path: "secrets/config_oauth.json"
-      status: "succeed"
-    - config_path: "integration_tests/invalid_config.json"
-      status: "failed"
-    - config_path: "integration_tests/invalid_config_oauth.json"
-      status: "failed"
-    - config_path: "integration_tests/invalid_config_wrong_title.json"
-      status: "failed"
+    tests:
+      - config_path: "secrets/config.json"
+        status: "succeed"
+      - config_path: "integration_tests/invalid_config.json"
+        status: "failed"
+      - config_path: "integration_tests/invalid_config_oauth.json"
+        status: "failed"
+      - config_path: "integration_tests/invalid_config_wrong_title.json"
+        status: "failed"
   discovery:
-    - config_path: "secrets/config.json"
-      backward_compatibility_tests_config:
-        disable_for_version: "0.1.83"
+    tests:
+      - config_path: "secrets/config.json"
+        backward_compatibility_tests_config:
+          # Quote stream is deleted so it is not backward compatible with previous discovery
+          disable_for_version: "0.2.3"
   basic_read:
-    - config_path: "secrets/config.json"
-      timeout_seconds: 600
-      configured_catalog_path: "sample_files/basic_read_catalog.json"
-      empty_streams: ["form_submissions", "ticket_pipelines", "engagements_meetings", "engagements_emails", "engagements", "engagements_calls", "quotes"]
-      # This test commented out, since it produces errors during active testing
-      # expect_records:
-      #   path: "integration_tests/expected_records.txt"
-    - config_path: "secrets/config_oauth.json"
-      timeout_seconds: 600
-      configured_catalog_path: "sample_files/basic_read_oauth_catalog.json"
-      empty_streams: ["form_submissions", "ticket_pipelines", "engagements_meetings", "engagements_emails", "engagements", "engagements_calls", "quotes"]
-      # This test commented out, since it produces errors during active testing
-      # expect_records:
-      #   path: "integration_tests/expected_records.txt"
+    tests:
+      - config_path: "secrets/config.json"
+        timeout_seconds: 600
+        configured_catalog_path: "sample_files/basic_read_oauth_catalog.json"
+        empty_streams:
+          - name: "form_submissions"
+            bypass_reason: "unable to populate"
+          - name: "form_submissions"
+            bypass_reason: "unable to populate"
+          - name: "ticket_pipelines"
+            bypass_reason: "unable to populate"
+          - name: "engagements_meetings"
+            bypass_reason: "unable to populate"
+          - name: "engagements_emails"
+            bypass_reason: "unable to populate"
+          - name: "engagements"
+            bypass_reason: "unable to populate"
+          - name: "engagements_calls"
+            bypass_reason: "unable to populate"
+          - name: "quotes"
+            bypass_reason: "unable to populate"
+        # This test commented out, since it produces errors during active testing
+        # expect_records:
+        #   path: "integration_tests/expected_records.txt"
   incremental:
-    - config_path: "secrets/config.json"
-      configured_catalog_path: "sample_files/incremental_catalog.json"
-      future_state_path: "integration_tests/abnormal_state.json"
+    tests:
+      - config_path: "secrets/config.json"
+        configured_catalog_path: "sample_files/incremental_catalog.json"
+        future_state:
+          future_state_path: "integration_tests/abnormal_state.json"
   full_refresh:
-    - config_path: "secrets/config.json"
-      configured_catalog_path: "sample_files/full_refresh_catalog.json"
-      ignored_fields:
-        "contact_lists": [ "properties", "ilsFilterBranch" ]
-        "companies": [ "properties", "hs_time_in_customer" ]
-        "companies": [ "properties", "hs_time_in_evangelist" ]
-        "companies": [ "properties", "hs_time_in_lead" ]
-        "companies": [ "properties", "hs_time_in_marketingqualifiedlead" ]
-        "companies": [ "properties", "hs_time_in_opportunity" ]
-        "companies": [ "properties", "hs_time_in_other" ]
-        "companies": [ "properties", "hs_time_in_salesqualifiedlead" ]
-        "companies": [ "properties", "hs_time_in_subscriber" ]
-        "contacts": [ "properties", "hs_time_in_customer" ]
-        "contacts": [ "properties", "hs_time_in_evangelist" ]
-        "contacts": [ "properties", "hs_time_in_lead" ]
-        "contacts": [ "properties", "hs_time_in_marketingqualifiedlead" ]
-        "contacts": [ "properties", "hs_time_in_opportunity" ]
-        "contacts": [ "properties", "hs_time_in_other" ]
-        "contacts": [ "properties", "hs_time_in_salesqualifiedlead" ]
-        "contacts": [ "properties", "hs_time_in_subscriber" ]
-        "deals": [ "properties", "hs_time_in_9567448" ]
-        "deals": [ "properties", "hs_time_in_9567449" ]
-        "deals": [ "properties", "hs_time_in_appointmentscheduled" ]
-        "deals": [ "properties", "hs_time_in_closedlost" ]
-        "deals": [ "properties", "hs_time_in_closedwon" ]
-        "deals": [ "properties", "hs_time_in_contractsent" ]
-        "deals": [ "properties", "hs_time_in_customclosedwonstage" ]
-        "deals": [ "properties", "hs_time_in_decisionmakerboughtin" ]
-        "deals": [ "properties", "hs_time_in_presentationscheduled" ]
-        "deals": [ "properties", "hs_time_in_qualifiedtobuy" ]
-        "tickets": [ "properties", "hs_time_in_1" ]
-        "tickets": [ "properties", "hs_time_in_2" ]
-        "tickets": [ "properties", "hs_time_in_3" ]
-        "tickets": [ "properties", "hs_time_in_4" ]
-    - config_path: "secrets/config_oauth.json"
-      configured_catalog_path: "sample_files/full_refresh_oauth_catalog.json"
-      ignored_fields:
-        "contact_lists": [ "properties", "ilsFilterBranch" ]
-        "companies": [ "properties", "hs_time_in_customer" ]
-        "companies": [ "properties", "hs_time_in_evangelist" ]
-        "companies": [ "properties", "hs_time_in_lead" ]
-        "companies": [ "properties", "hs_time_in_marketingqualifiedlead" ]
-        "companies": [ "properties", "hs_time_in_opportunity" ]
-        "companies": [ "properties", "hs_time_in_other" ]
-        "companies": [ "properties", "hs_time_in_salesqualifiedlead" ]
-        "companies": [ "properties", "hs_time_in_subscriber" ]
-        "contacts": [ "properties", "hs_time_in_customer" ]
-        "contacts": [ "properties", "hs_time_in_evangelist" ]
-        "contacts": [ "properties", "hs_time_in_lead" ]
-        "contacts": [ "properties", "hs_time_in_marketingqualifiedlead" ]
-        "contacts": [ "properties", "hs_time_in_opportunity" ]
-        "contacts": [ "properties", "hs_time_in_other" ]
-        "contacts": [ "properties", "hs_time_in_salesqualifiedlead" ]
-        "contacts": [ "properties", "hs_time_in_subscriber" ]
-        "deals": [ "properties", "hs_time_in_9567448" ]
-        "deals": [ "properties", "hs_time_in_9567449" ]
-        "deals": [ "properties", "hs_time_in_appointmentscheduled" ]
-        "deals": [ "properties", "hs_time_in_closedlost" ]
-        "deals": [ "properties", "hs_time_in_closedwon" ]
-        "deals": [ "properties", "hs_time_in_contractsent" ]
-        "deals": [ "properties", "hs_time_in_customclosedwonstage" ]
-        "deals": [ "properties", "hs_time_in_decisionmakerboughtin" ]
-        "deals": [ "properties", "hs_time_in_presentationscheduled" ]
-        "deals": [ "properties", "hs_time_in_qualifiedtobuy" ]
-        "tickets": [ "properties", "hs_time_in_1" ]
-        "tickets": [ "properties", "hs_time_in_2" ]
-        "tickets": [ "properties", "hs_time_in_3" ]
-        "tickets": [ "properties", "hs_time_in_4" ]
+    tests:
+      - config_path: "secrets/config.json"
+        configured_catalog_path: "sample_files/full_refresh_oauth_catalog.json"
+        ignored_fields:
+          "contact_lists": [ "properties", "ilsFilterBranch" ]
+          "companies": [ "properties", "hs_time_in_customer" ]
+          "companies": [ "properties", "hs_time_in_evangelist" ]
+          "companies": [ "properties", "hs_time_in_lead" ]
+          "companies": [ "properties", "hs_time_in_marketingqualifiedlead" ]
+          "companies": [ "properties", "hs_time_in_opportunity" ]
+          "companies": [ "properties", "hs_time_in_other" ]
+          "companies": [ "properties", "hs_time_in_salesqualifiedlead" ]
+          "companies": [ "properties", "hs_time_in_subscriber" ]
+          "contacts": [ "properties", "hs_time_in_customer" ]
+          "contacts": [ "properties", "hs_time_in_evangelist" ]
+          "contacts": [ "properties", "hs_time_in_lead" ]
+          "contacts": [ "properties", "hs_time_in_marketingqualifiedlead" ]
+          "contacts": [ "properties", "hs_time_in_opportunity" ]
+          "contacts": [ "properties", "hs_time_in_other" ]
+          "contacts": [ "properties", "hs_time_in_salesqualifiedlead" ]
+          "contacts": [ "properties", "hs_time_in_subscriber" ]
+          "deals": [ "properties", "hs_time_in_9567448" ]
+          "deals": [ "properties", "hs_time_in_9567449" ]
+          "deals": [ "properties", "hs_time_in_appointmentscheduled" ]
+          "deals": [ "properties", "hs_time_in_closedlost" ]
+          "deals": [ "properties", "hs_time_in_closedwon" ]
+          "deals": [ "properties", "hs_time_in_contractsent" ]
+          "deals": [ "properties", "hs_time_in_customclosedwonstage" ]
+          "deals": [ "properties", "hs_time_in_decisionmakerboughtin" ]
+          "deals": [ "properties", "hs_time_in_presentationscheduled" ]
+          "deals": [ "properties", "hs_time_in_qualifiedtobuy" ]
+          "tickets": [ "properties", "hs_time_in_1" ]
+          "tickets": [ "properties", "hs_time_in_2" ]
+          "tickets": [ "properties", "hs_time_in_3" ]
+          "tickets": [ "properties", "hs_time_in_4" ]
diff --git a/airbyte-integrations/connectors/source-hubspot/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-hubspot/acceptance-test-docker.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env sh
 
 # Build latest connector image
-docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2):dev
+docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-):dev
 
 # Pull latest acctest image
 docker pull airbyte/source-acceptance-test:latest

diff --git a/airbyte-integrations/connectors/source-hubspot/integration_tests/test_associations.py b/airbyte-integrations/connectors/source-hubspot/integration_tests/test_associations.py
@@ -37,10 +37,8 @@ def configured_catalog(config, source):
     }
 
 
-@pytest.mark.parametrize("auth", ("api_key", "oauth"))
-def test_incremental_read_fetches_associations(auth, config, oauth_config, configured_catalog, source, associations):
-    configuration = oauth_config if auth == "oauth" else config
-    messages = source.read(logging.getLogger("airbyte"), configuration, ConfiguredAirbyteCatalog.parse_obj(configured_catalog), {})
+def test_incremental_read_fetches_associations(config, configured_catalog, source, associations):
+    messages = source.read(logging.getLogger("airbyte"), config, ConfiguredAirbyteCatalog.parse_obj(configured_catalog), {})
 
     association_found = False
     for message in messages:

diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/basic_read_catalog.json b/airbyte-integrations/connectors/source-hubspot/sample_files/basic_read_catalog.json
@@ -224,18 +224,6 @@
       "cursor_field": ["timestamp"],
       "destination_sync_mode": "overwrite"
     },
-    {
-      "stream": {
-        "name": "quotes",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["updatedAt"]
-      },
-      "sync_mode": "full_refresh",
-      "cursor_field": ["updatedAt"],
-      "destination_sync_mode": "append"
-    },
     {
       "stream": {
         "name": "subscription_changes",

diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/full_refresh_catalog.json b/airbyte-integrations/connectors/source-hubspot/sample_files/full_refresh_catalog.json
@@ -117,15 +117,6 @@
       "sync_mode": "full_refresh",
       "destination_sync_mode": "overwrite"
     },
-    {
-      "stream": {
-        "name": "quotes",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh"]
-      },
-      "sync_mode": "full_refresh",
-      "destination_sync_mode": "overwrite"
-    },
     {
       "stream": {
         "name": "tickets",

diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/incremental_catalog.json b/airbyte-integrations/connectors/source-hubspot/sample_files/incremental_catalog.json
@@ -213,18 +213,6 @@
       "cursor_field": ["updatedAt"],
       "destination_sync_mode": "append"
     },
-    {
-      "stream": {
-        "name": "quotes",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"],
-        "source_defined_cursor": true,
-        "default_cursor_field": ["updatedAt"]
-      },
-      "sync_mode": "incremental",
-      "cursor_field": ["updatedAt"],
-      "destination_sync_mode": "append"
-    },
     {
       "stream": {
         "name": "subscription_changes",

diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/sample_state.json b/airbyte-integrations/connectors/source-hubspot/sample_files/sample_state.json
@@ -23,9 +23,6 @@
   "products": {
     "updatedAt": "2021-02-23T00:00:00Z"
   },
-  "quotes": {
-    "updatedAt": "2021-02-23T00:00:00Z"
-  },
   "subscription_changes": {
     "timestamp": "2021-02-23T00:00:00Z"
   },

diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/constants.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/constants.py
@@ -4,4 +4,3 @@
 
 OAUTH_CREDENTIALS = "OAuth Credentials"
 PRIVATE_APP_CREDENTIALS = "Private App Credentials"
-API_KEY_CREDENTIALS = "API Key Credentials"
diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/quotes.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/quotes.json
diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py
@@ -16,7 +16,6 @@
 from airbyte_cdk.utils.event_timing import create_timer
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 from requests import HTTPError
-from source_hubspot.constants import API_KEY_CREDENTIALS
 from source_hubspot.streams import (
     API,
     Campaigns,
@@ -40,7 +39,6 @@
     Owners,
     Products,
     PropertyHistory,
-    Quotes,
     SubscriptionChanges,
     TicketPipelines,
     Tickets,
@@ -118,10 +116,6 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
             Workflows(**common_params),
         ]
 
-        credentials_title = credentials.get("credentials_title")
-        if credentials_title == API_KEY_CREDENTIALS:
-            streams.append(Quotes(**common_params))
-
         api = API(credentials=credentials)
         if api.is_oauth2():
             authenticator = API(credentials=credentials).get_authenticator()