From b03b1823124bc62fedddccfa0d3f62caa70a6ece Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Fri, 17 Feb 2023 11:53:19 +0200 Subject: [PATCH 1/5] #20703 source salesforce: include primary key in every chunk --- .../connectors/source-salesforce/Dockerfile | 2 +- .../source-salesforce/source_salesforce/streams.py | 7 +++++-- .../connectors/source-salesforce/unit_tests/api_test.py | 9 ++++++--- docs/integrations/sources/salesforce.md | 1 + 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/airbyte-integrations/connectors/source-salesforce/Dockerfile b/airbyte-integrations/connectors/source-salesforce/Dockerfile index a8482a934c280..905a28e0e7cbd 100644 --- a/airbyte-integrations/connectors/source-salesforce/Dockerfile +++ b/airbyte-integrations/connectors/source-salesforce/Dockerfile @@ -13,5 +13,5 @@ RUN pip install . ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=2.0.2 +LABEL io.airbyte.version=2.0.3 LABEL io.airbyte.name=airbyte/source-salesforce diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py index b5d7434462b0f..82cde6f41b7d0 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py @@ -136,13 +136,16 @@ def request_params( def chunk_properties(self) -> Iterable[Mapping[str, Any]]: selected_properties = self.get_json_schema().get("properties", {}) + def empty_props(): + return {self.primary_key: selected_properties[self.primary_key]} if self.primary_key else {} + summary_length = 0 - local_properties = {} + local_properties = empty_props() for property_name, value in selected_properties.items(): current_property_length = len(urllib.parse.quote(f"{property_name},")) if current_property_length + summary_length >= self.max_properties_length: yield local_properties - local_properties = {} + local_properties = empty_props() summary_length = 0 local_properties[property_name] = value diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py index 7bffa1ceaaf06..2cf602b1373bb 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py @@ -627,7 +627,10 @@ def test_rest_stream_init_with_too_many_properties(stream_config, stream_api_v2_ def test_too_many_properties(stream_config, stream_api_v2_pk_too_many_properties, requests_mock): stream = generate_stream("Account", stream_config, stream_api_v2_pk_too_many_properties) - chunks = len(list(stream.chunk_properties())) + chunks = list(stream.chunk_properties()) + for chunk in chunks: + assert stream.primary_key in chunk + chunks_len = len(chunks) assert stream.too_many_properties assert stream.primary_key assert type(stream) == RestSalesforceStream @@ -648,7 +651,7 @@ def test_too_many_properties(stream_config, stream_api_v2_pk_too_many_properties } }, # 2 for 2 chunks above and 1 for a chunk below - *[{"json": {"records": [{"Id": 1}, {"Id": 2}], "nextRecordsUrl": next_page_url}} for _ in range(chunks - 3)], + *[{"json": {"records": [{"Id": 1}, {"Id": 2}], "nextRecordsUrl": next_page_url}} for _ in range(chunks_len - 3)], { "json": { "records": [{"Id": 1}, {"Id": 2}] @@ -665,7 +668,7 @@ def test_too_many_properties(stream_config, stream_api_v2_pk_too_many_properties } }, # 2 for 2 chunks above and 1 for a chunk below - *[{"json": {"records": [{"Id": 3}, {"Id": 4}]}} for _ in range(chunks - 3)], + *[{"json": {"records": [{"Id": 3}, {"Id": 4}]}} for _ in range(chunks_len - 3)], { "json": { "records": [{"Id": 3}, {"Id": 4}] diff --git a/docs/integrations/sources/salesforce.md b/docs/integrations/sources/salesforce.md index 5767045d4ab82..3ee4437a91548 100644 --- a/docs/integrations/sources/salesforce.md +++ b/docs/integrations/sources/salesforce.md @@ -129,6 +129,7 @@ Now that you have set up the Salesforce source connector, check out the followin | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------| +| 2.0.3 | 2023-02-17 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | In case properties are chunked, fetch primary key in every chunk | | 2.0.2 | 2023-02-13 | [22896](https://github.com/airbytehq/airbyte/pull/22896) | Count the URL length based on encoded params | | 2.0.1 | 2023-02-08 | [22597](https://github.com/airbytehq/airbyte/pull/22597) | Make multiple requests if a REST stream has too many properties | | 2.0.0 | 2023-02-02 | [22322](https://github.com/airbytehq/airbyte/pull/22322) | Remove `ActivityMetricRollup` stream | From f889f34dca3fa51c0417fe167972d394841fc9a1 Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Fri, 17 Feb 2023 12:00:32 +0200 Subject: [PATCH 2/5] #20703 source salesforce: upd changelog --- docs/integrations/sources/salesforce.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/salesforce.md b/docs/integrations/sources/salesforce.md index 3ee4437a91548..106c12941f9ce 100644 --- a/docs/integrations/sources/salesforce.md +++ b/docs/integrations/sources/salesforce.md @@ -129,7 +129,7 @@ Now that you have set up the Salesforce source connector, check out the followin | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------| -| 2.0.3 | 2023-02-17 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | In case properties are chunked, fetch primary key in every chunk | +| 2.0.3 | 2023-02-17 | [23190](https://github.com/airbytehq/airbyte/pull/23190) | In case properties are chunked, fetch primary key in every chunk | | 2.0.2 | 2023-02-13 | [22896](https://github.com/airbytehq/airbyte/pull/22896) | Count the URL length based on encoded params | | 2.0.1 | 2023-02-08 | [22597](https://github.com/airbytehq/airbyte/pull/22597) | Make multiple requests if a REST stream has too many properties | | 2.0.0 | 2023-02-02 | [22322](https://github.com/airbytehq/airbyte/pull/22322) | Remove `ActivityMetricRollup` stream | From 1af92020e2ea7be8c72e51d9ec3ece5c3ac885e0 Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Mon, 20 Feb 2023 13:33:35 +0200 Subject: [PATCH 3/5] #20703 source salesforce: review fix --- .../source-salesforce/source_salesforce/streams.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py index 82cde6f41b7d0..92159756c3f65 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py @@ -136,16 +136,16 @@ def request_params( def chunk_properties(self) -> Iterable[Mapping[str, Any]]: selected_properties = self.get_json_schema().get("properties", {}) - def empty_props(): + def empty_props_with_pk_if_present(): return {self.primary_key: selected_properties[self.primary_key]} if self.primary_key else {} summary_length = 0 - local_properties = empty_props() + local_properties = empty_props_with_pk_if_present() for property_name, value in selected_properties.items(): current_property_length = len(urllib.parse.quote(f"{property_name},")) if current_property_length + summary_length >= self.max_properties_length: yield local_properties - local_properties = empty_props() + local_properties = empty_props_with_pk_if_present() summary_length = 0 local_properties[property_name] = value From b45e4a9b40799ee7815e35c06297c65c6a92e8e4 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Thu, 23 Feb 2023 07:55:12 +0000 Subject: [PATCH 4/5] auto-bump connector version --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 35e40f73dc544..f5210546a0cf2 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1574,7 +1574,7 @@ - name: Salesforce sourceDefinitionId: b117307c-14b6-41aa-9422-947e34922962 dockerRepository: airbyte/source-salesforce - dockerImageTag: 2.0.2 + dockerImageTag: 2.0.3 documentationUrl: https://docs.airbyte.com/integrations/sources/salesforce icon: salesforce.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 00164009adf69..7ded6d807a5b7 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -13106,7 +13106,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-salesforce:2.0.2" +- dockerImage: "airbyte/source-salesforce:2.0.3" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/salesforce" connectionSpecification: From eaadb13b8666e3f6b32d5ffd3ae0abf5afa6d930 Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Thu, 23 Feb 2023 19:54:41 +0200 Subject: [PATCH 5/5] #20703 source salesforce: upd connectors.md --- connectors.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/connectors.md b/connectors.md index bcab9fd251cc1..72af95e6706c1 100644 --- a/connectors.md +++ b/connectors.md @@ -200,7 +200,7 @@ | **SFTP** | SFTP icon | Source | airbyte/source-sftp:0.1.2 | alpha | [link](https://docs.airbyte.com/integrations/sources/sftp) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-sftp) | `a827c52e-791c-4135-a245-e233c5255199` | | **SFTP Bulk** | SFTP Bulk icon | Source | airbyte/source-sftp-bulk:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/sftp-bulk) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-sftp-bulk) | `31e3242f-dee7-4cdc-a4b8-8e06c5458517` | | **SalesLoft** | SalesLoft icon | Source | airbyte/source-salesloft:0.1.3 | alpha | [link](https://docs.airbyte.com/integrations/sources/salesloft) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-salesloft) | `41991d12-d4b5-439e-afd0-260a31d4c53f` | -| **Salesforce** | Salesforce icon | Source | airbyte/source-salesforce:2.0.2 | generally_available | [link](https://docs.airbyte.com/integrations/sources/salesforce) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-salesforce) | `b117307c-14b6-41aa-9422-947e34922962` | +| **Salesforce** | Salesforce icon | Source | airbyte/source-salesforce:2.0.3 | generally_available | [link](https://docs.airbyte.com/integrations/sources/salesforce) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-salesforce) | `b117307c-14b6-41aa-9422-947e34922962` | | **Sample Data (Faker)** | Sample Data (Faker) icon | Source | airbyte/source-faker:2.0.3 | beta | [link](https://docs.airbyte.com/integrations/sources/faker) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-faker) | `dfd88b22-b603-4c3d-aad7-3701784586b1` | | **SearchMetrics** | SearchMetrics icon | Source | airbyte/source-search-metrics:0.1.1 | alpha | [link](https://docs.airbyte.com/integrations/sources/search-metrics) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-search-metrics) | `8d7ef552-2c0f-11ec-8d3d-0242ac130003` | | **Secoda** | Secoda icon | Source | airbyte/source-secoda:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/secoda) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-secoda) | `da9fc6b9-8059-4be0-b204-f56e22e4d52d` |