🐛Source Hubspot: Fixed 403 error validation (#16914)

* Fixed 403 error validation * Updated PR number * Updated after review * Updated after review * Updated version * Add ignoredd fields * auto-bump connector version [ci skip] Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
airbytehq · Oct 3, 2022 · 06186d2 · 06186d2
1 parent f94185e
commit 06186d2
Show file tree

Hide file tree

Showing 8 changed files with 25 additions and 18 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -472,7 +472,7 @@
 - name: HubSpot
   sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c
   dockerRepository: airbyte/source-hubspot
-  dockerImageTag: 0.2.1
+  dockerImageTag: 0.2.2
   documentationUrl: https://docs.airbyte.io/integrations/sources/hubspot
   icon: hubspot.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -4715,7 +4715,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-hubspot:0.2.1"
+- dockerImage: "airbyte/source-hubspot:0.2.2"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/hubspot"
     connectionSpecification:

diff --git a/airbyte-integrations/connectors/source-hubspot/Dockerfile b/airbyte-integrations/connectors/source-hubspot/Dockerfile
@@ -34,5 +34,5 @@ COPY source_hubspot ./source_hubspot
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.1
+LABEL io.airbyte.version=0.2.2
 LABEL io.airbyte.name=airbyte/source-hubspot
diff --git a/airbyte-integrations/connectors/source-hubspot/README.md b/airbyte-integrations/connectors/source-hubspot/README.md
@@ -99,7 +99,7 @@ python main.py read --config secrets/config.json --catalog sample_files/configur
 Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. First install test dependencies into your virtual environment:
 
 ```
-pip install .[tests]
+pip install .'[tests]'
 ```
 
 ### Unit Tests

diff --git a/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml b/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml
@@ -40,6 +40,7 @@ tests:
     - config_path: "secrets/config.json"
       configured_catalog_path: "sample_files/full_refresh_catalog.json"
       ignored_fields:
+        "contact_lists": [ "properties", "ilsFilterBranch" ]
         "companies": [ "properties", "hs_time_in_customer" ]
         "companies": [ "properties", "hs_time_in_evangelist" ]
         "companies": [ "properties", "hs_time_in_lead" ]
@@ -73,6 +74,7 @@ tests:
     - config_path: "secrets/config_oauth.json"
       configured_catalog_path: "sample_files/full_refresh_oauth_catalog.json"
       ignored_fields:
+        "contact_lists": [ "properties", "ilsFilterBranch" ]
         "companies": [ "properties", "hs_time_in_customer" ]
         "companies": [ "properties", "hs_time_in_evangelist" ]
         "companies": [ "properties", "hs_time_in_lead" ]

diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py
@@ -208,6 +208,7 @@ class Stream(HttpStream, ABC):
     primary_key = None
     filter_old_records: bool = True
     denormalize_records: bool = False  # one record from API response can result in multiple records emitted
+    raise_on_http_errors: bool = True
 
     @property
     @abstractmethod
@@ -259,6 +260,12 @@ def __init__(self, api: API, start_date: Union[str, pendulum.datetime], credenti
         elif creds_title in (OAUTH_CREDENTIALS, PRIVATE_APP_CREDENTIALS):
             self._authenticator = api.get_authenticator()
 
+    def should_retry(self, response: requests.Response) -> bool:
+        if response.status_code == HTTPStatus.FORBIDDEN:
+            setattr(self, "raise_on_http_errors", False)
+            logger.warning("You have not permission to API for this stream. " "Please check your scopes for Hubspot account.")
+        return super().should_retry(response)
+
     def backoff_time(self, response: requests.Response) -> Optional[float]:
         if response.status_code == codes.too_many_requests:
             return float(response.headers.get("Retry-After", 3))

diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py
@@ -142,15 +142,11 @@ def test_wrong_permissions_api_key(requests_mock, creds_with_wrong_permissions,
     """
 
     # Mapping tipical response for mocker
-    responses = [
-        {
-            "json": {
-                "status": "error",
-                "message": f'This hapikey ({creds_with_wrong_permissions.get("api_key")}) does not have proper permissions! (requires any of [automation-access])',
-                "correlationId": "2fe0a9af-3609-45c9-a4d7-83a1774121aa",
-            }
-        }
-    ]
+    json = {
+        "status": "error",
+        "message": f'This hapikey ({creds_with_wrong_permissions.get("api_key")}) does not have proper permissions! (requires any of [automation-access])',
+        "correlationId": "2fe0a9af-3609-45c9-a4d7-83a1774121aa",
+    }
 
     # We expect something like this
     expected_warining_message = {
@@ -165,11 +161,12 @@ def test_wrong_permissions_api_key(requests_mock, creds_with_wrong_permissions,
     test_stream = Workflows(**common_params)
 
     # Mocking Request
-    requests_mock.register_uri("GET", test_stream.url, responses)
-    list(test_stream.read_records(sync_mode=SyncMode.full_refresh))
+    requests_mock.register_uri("GET", test_stream.url, json=json, status_code=403)
+    records = list(test_stream.read_records(sync_mode=SyncMode.full_refresh))
 
     # match logged expected logged warning message with output given from preudo-output
     assert expected_warining_message["log"]["message"] in caplog.text
+    assert not records
 
 
 class TestSplittingPropertiesFunctionality:
@@ -354,7 +351,7 @@ def test_search_based_stream_should_not_attempt_to_get_more_than_10k_records(req
                 "results": [{"id": f"{y}", "updatedAt": "2022-02-25T16:43:11Z"} for y in range(100)],
                 "paging": {
                     "next": {
-                        "after": f"{x*100}",
+                        "after": f"{x * 100}",
                     }
                 },
             },
@@ -370,7 +367,7 @@ def test_search_based_stream_should_not_attempt_to_get_more_than_10k_records(req
                     "results": [{"id": f"{y}", "updatedAt": "2022-03-01T00:00:00Z"} for y in range(100)],
                     "paging": {
                         "next": {
-                            "after": f"{x*100}",
+                            "after": f"{x * 100}",
                         }
                     },
                 },

diff --git a/docs/integrations/sources/hubspot.md b/docs/integrations/sources/hubspot.md
@@ -131,7 +131,8 @@ Now that you have set up the Hubspot source connector, check out the following H
 ## Changelog
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                                                        |
-| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------- |
+|:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.2.2   | 2022-10-03 | [16914](https://github.com/airbytehq/airbyte/pull/16914) | Fix 403 forbidden error validation                                                                                                             |
 | 0.2.1   | 2022-09-26 | [17120](https://github.com/airbytehq/airbyte/pull/17120) | Migrate to per-stream state.                                                                                                                   |
 | 0.2.0   | 2022-09-13 | [16632](https://github.com/airbytehq/airbyte/pull/16632) | Remove Feedback Submissions stream as the one using unstable (beta) API.                                                                       |
 | 0.1.83  | 2022-09-01 | [16214](https://github.com/airbytehq/airbyte/pull/16214) | Update Tickets, fix missing properties and change how state is updated.                                                                        |