#20703 source salesforce: include primary key in every chunk

airbytehq · Feb 17, 2023 · b03b182 · b03b182
1 parent 2ce3c17
commit b03b182
Show file tree

Hide file tree

Showing 4 changed files with 13 additions and 6 deletions.
diff --git a/airbyte-integrations/connectors/source-salesforce/Dockerfile b/airbyte-integrations/connectors/source-salesforce/Dockerfile
@@ -13,5 +13,5 @@ RUN pip install .
 
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=2.0.2
+LABEL io.airbyte.version=2.0.3
 LABEL io.airbyte.name=airbyte/source-salesforce
diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py
@@ -136,13 +136,16 @@ def request_params(
     def chunk_properties(self) -> Iterable[Mapping[str, Any]]:
         selected_properties = self.get_json_schema().get("properties", {})
 
+        def empty_props():
+            return {self.primary_key: selected_properties[self.primary_key]} if self.primary_key else {}
+
         summary_length = 0
-        local_properties = {}
+        local_properties = empty_props()
         for property_name, value in selected_properties.items():
             current_property_length = len(urllib.parse.quote(f"{property_name},"))
             if current_property_length + summary_length >= self.max_properties_length:
                 yield local_properties
-                local_properties = {}
+                local_properties = empty_props()
                 summary_length = 0
 
             local_properties[property_name] = value

diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py
@@ -627,7 +627,10 @@ def test_rest_stream_init_with_too_many_properties(stream_config, stream_api_v2_
 
 def test_too_many_properties(stream_config, stream_api_v2_pk_too_many_properties, requests_mock):
     stream = generate_stream("Account", stream_config, stream_api_v2_pk_too_many_properties)
-    chunks = len(list(stream.chunk_properties()))
+    chunks = list(stream.chunk_properties())
+    for chunk in chunks:
+        assert stream.primary_key in chunk
+    chunks_len = len(chunks)
     assert stream.too_many_properties
     assert stream.primary_key
     assert type(stream) == RestSalesforceStream
@@ -648,7 +651,7 @@ def test_too_many_properties(stream_config, stream_api_v2_pk_too_many_properties
                 }
             },
             # 2 for 2 chunks above and 1 for a chunk below
-            *[{"json": {"records": [{"Id": 1}, {"Id": 2}], "nextRecordsUrl": next_page_url}} for _ in range(chunks - 3)],
+            *[{"json": {"records": [{"Id": 1}, {"Id": 2}], "nextRecordsUrl": next_page_url}} for _ in range(chunks_len - 3)],
             {
                 "json": {
                     "records": [{"Id": 1}, {"Id": 2}]
@@ -665,7 +668,7 @@ def test_too_many_properties(stream_config, stream_api_v2_pk_too_many_properties
                 }
             },
             # 2 for 2 chunks above and 1 for a chunk below
-            *[{"json": {"records": [{"Id": 3}, {"Id": 4}]}} for _ in range(chunks - 3)],
+            *[{"json": {"records": [{"Id": 3}, {"Id": 4}]}} for _ in range(chunks_len - 3)],
             {
                 "json": {
                     "records": [{"Id": 3}, {"Id": 4}]

diff --git a/docs/integrations/sources/salesforce.md b/docs/integrations/sources/salesforce.md
@@ -129,6 +129,7 @@ Now that you have set up the Salesforce source connector, check out the followin
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                                          |
 |:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------|
+| 2.0.3   | 2023-02-17 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | In case properties are chunked, fetch primary key in every chunk                                                                 |
 | 2.0.2   | 2023-02-13 | [22896](https://github.com/airbytehq/airbyte/pull/22896) | Count the URL length based on encoded params                                                                                     |
 | 2.0.1   | 2023-02-08 | [22597](https://github.com/airbytehq/airbyte/pull/22597) | Make multiple requests if a REST stream has too many properties                                                                  |
 | 2.0.0   | 2023-02-02 | [22322](https://github.com/airbytehq/airbyte/pull/22322) | Remove `ActivityMetricRollup` stream                                                                                             |