Source Hubspot: fix URI too long issue (#13691)

* #268: fix URI too long (HubSpot) * #268 upd changelog * #268 oncall: review fix * auto-bump connector version Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
airbytehq · Jun 10, 2022 · 5df20b7 · 5df20b7
1 parent 6c8cae2
commit 5df20b7
Show file tree

Hide file tree

Showing 6 changed files with 30 additions and 5 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -390,7 +390,7 @@
 - name: HubSpot
   sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c
   dockerRepository: airbyte/source-hubspot
-  dockerImageTag: 0.1.68
+  dockerImageTag: 0.1.69
   documentationUrl: https://docs.airbyte.io/integrations/sources/hubspot
   icon: hubspot.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -3637,7 +3637,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-hubspot:0.1.68"
+- dockerImage: "airbyte/source-hubspot:0.1.69"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/hubspot"
     connectionSpecification:

diff --git a/airbyte-integrations/connectors/source-hubspot/Dockerfile b/airbyte-integrations/connectors/source-hubspot/Dockerfile
@@ -34,5 +34,5 @@ COPY source_hubspot ./source_hubspot
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.68
+LABEL io.airbyte.version=0.1.69
 LABEL io.airbyte.name=airbyte/source-hubspot
diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py
@@ -64,13 +64,14 @@ def split_properties(properties_list: List[str]) -> Iterator[Tuple[str]]:
     summary_length = 0
     local_properties = []
     for property_ in properties_list:
-        if len(property_) + summary_length + len(urllib.parse.quote(",")) >= PROPERTIES_PARAM_MAX_LENGTH:
+        current_property_length = len(urllib.parse.quote(f"property={property_}&"))
+        if current_property_length + summary_length >= PROPERTIES_PARAM_MAX_LENGTH:
             yield local_properties
             local_properties = []
             summary_length = 0
 
         local_properties.append(property_)
-        summary_length += len(property_) + len(urllib.parse.quote(","))
+        summary_length += current_property_length
 
     if local_properties:
         yield local_properties

diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_split_properties.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_split_properties.py
@@ -0,0 +1,23 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+import pytest
+from source_hubspot.streams import split_properties
+
+lorem_ipsum = """Lorem ipsum dolor sit amet, consectetur adipiscing elit"""
+lorem_ipsum = lorem_ipsum.lower().replace(",", "")
+
+many_properties = lorem_ipsum.split(" ") * 100
+few_properties = ["firstname", "lastname", "age", "dob", "id"]
+
+
+@pytest.mark.parametrize(("properties", "chunks_expected"), ((few_properties, 1), (many_properties, 2)))
+def test_split_properties(properties, chunks_expected):
+    chunked_properties = set()
+    index = 0
+    for index, chunk in enumerate(split_properties(properties)):
+        chunked_properties |= set(chunk)
+    chunks = index + 1
+    assert chunked_properties == set(properties)
+    assert chunks == chunks_expected
diff --git a/docs/integrations/sources/hubspot.md b/docs/integrations/sources/hubspot.md
@@ -133,6 +133,7 @@ HubSpot's API will [rate limit](https://developers.hubspot.com/docs/api/usage-de
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                                                        |
 |:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.1.69  | 2022-06-10 | [13691](https://github.com/airbytehq/airbyte/pull/13691) | Fix the `URI Too Long` issue                                                                                                                   |
 | 0.1.68  | 2022-06-08 | [13596](https://github.com/airbytehq/airbyte/pull/13596) | Fix for the `property_history` which did not emit records                                                                                      |
 | 0.1.67  | 2022-06-07 | [13566](https://github.com/airbytehq/airbyte/pull/13566) | Report which scopes are missing to the user                                                                                                    |
 | 0.1.66  | 2022-06-05 | [13475](https://github.com/airbytehq/airbyte/pull/13475) | Scope `crm.objects.feedback_submissions.read` added for `feedback_submissions` stream                                                          |