From b5f12c2ef6424dbff29e11efa0a29795490a33f1 Mon Sep 17 00:00:00 2001 From: Arsen Losenko <20901439+arsenlosenko@users.noreply.github.com> Date: Fri, 11 Aug 2023 19:08:16 +0300 Subject: [PATCH] Source Hubspot: Add OwnersArchived stream (#29249) * Source Hubspot: Add OwnersArchived stream * Update changelog and CAT config * Update expected records, update schema * Update unit tests * Set archived to true explicitly --- .../connectors/source-hubspot/Dockerfile | 2 +- .../source-hubspot/acceptance-test-config.yml | 2 + .../connectors/source-hubspot/metadata.yaml | 4 +- .../sample_files/basic_read_catalog.json | 9 ++++ .../schemas/owners_archived.json | 49 +++++++++++++++++++ .../source-hubspot/source_hubspot/source.py | 2 + .../source-hubspot/source_hubspot/streams.py | 24 +++++++-- .../source-hubspot/unit_tests/test_source.py | 2 +- .../source-hubspot/unit_tests/test_streams.py | 2 + docs/integrations/sources/hubspot.md | 1 + 10 files changed, 90 insertions(+), 7 deletions(-) create mode 100644 airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/owners_archived.json diff --git a/airbyte-integrations/connectors/source-hubspot/Dockerfile b/airbyte-integrations/connectors/source-hubspot/Dockerfile index 863dd6c768d8b..ab13c4613a23a 100644 --- a/airbyte-integrations/connectors/source-hubspot/Dockerfile +++ b/airbyte-integrations/connectors/source-hubspot/Dockerfile @@ -34,5 +34,5 @@ COPY source_hubspot ./source_hubspot ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.3.3 +LABEL io.airbyte.version=1.4.0 LABEL io.airbyte.name=airbyte/source-hubspot diff --git a/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml b/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml index 1d34c781edf9f..94fa9c282e4c4 100644 --- a/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml @@ -41,6 +41,8 @@ acceptance_tests: bypass_reason: Unable to populate - name: deals_archived bypass_reason: Unable to populate + - name: owners_archived + bypass_reason: unable to populate ignored_fields: contact_lists: - name: ilsFilterBranch diff --git a/airbyte-integrations/connectors/source-hubspot/metadata.yaml b/airbyte-integrations/connectors/source-hubspot/metadata.yaml index 27690ba25ba3d..726849a497583 100644 --- a/airbyte-integrations/connectors/source-hubspot/metadata.yaml +++ b/airbyte-integrations/connectors/source-hubspot/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c - dockerImageTag: 1.3.3 + dockerImageTag: 1.4.0 dockerRepository: airbyte/source-hubspot githubIssueLabel: source-hubspot icon: hubspot.svg @@ -14,7 +14,7 @@ data: registries: cloud: enabled: true - dockerImageTag: 1.3.3 + dockerImageTag: 1.4.0 oss: enabled: true releaseStage: generally_available diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/basic_read_catalog.json b/airbyte-integrations/connectors/source-hubspot/sample_files/basic_read_catalog.json index 068be3416c05f..90d38d0e6d124 100644 --- a/airbyte-integrations/connectors/source-hubspot/sample_files/basic_read_catalog.json +++ b/airbyte-integrations/connectors/source-hubspot/sample_files/basic_read_catalog.json @@ -222,6 +222,15 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, + { + "stream": { + "name": "owners_archived", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "products", diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/owners_archived.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/owners_archived.json new file mode 100644 index 0000000000000..15150ec585d3f --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/owners_archived.json @@ -0,0 +1,49 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "firstName": { + "type": ["null", "string"] + }, + "lastName": { + "type": ["null", "string"] + }, + "userId": { + "type": ["null", "integer"] + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "updatedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "archived": { + "type": ["null", "boolean"] + }, + "teams": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "membership": { + "type": ["null", "string"] + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py index 67f65d735d510..2d1c6a642dca3 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py @@ -39,6 +39,7 @@ LineItems, MarketingEmails, Owners, + OwnersArchived, Products, PropertyHistory, SubscriptionChanges, @@ -120,6 +121,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: LineItems(**common_params), MarketingEmails(**common_params), Owners(**common_params), + OwnersArchived(**common_params), Products(**common_params), PropertyHistory(**common_params), SubscriptionChanges(**common_params), diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py index 98c1ea690a3ad..d272bff24316b 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py @@ -1552,7 +1552,6 @@ def read_records( stream_slice: Mapping[str, Any] = None, stream_state: Mapping[str, Any] = None, ) -> Iterable[Mapping[str, Any]]: - self.latest_cursor = None # The date we need records since @@ -1693,6 +1692,27 @@ class Owners(ClientSideIncrementalStream): scopes = {"crm.objects.owners.read"} +class OwnersArchived(ClientSideIncrementalStream): + """Archived Owners, API v3""" + + url = "/crm/v3/owners" + updated_at_field = "updatedAt" + created_at_field = "createdAt" + cursor_field_datetime_format = "YYYY-MM-DDTHH:mm:ss.SSSSSSZ" + primary_key = "id" + scopes = {"crm.objects.owners.read"} + + def request_params( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + params = super().request_params(stream_state, stream_slice, next_page_token) + params["archived"] = "true" + return params + + class PropertyHistory(Stream): """Contacts Endpoint, API v1 Is used to get all Contacts and the history of their respective @@ -1790,7 +1810,6 @@ class Contacts(CRMSearchStream): class ContactsMergedAudit(Stream): - url = "/contacts/v1/contact/vids/batch/" updated_at_field = "timestamp" scopes = {"crm.objects.contacts.read"} @@ -1810,7 +1829,6 @@ def get_json_schema(self) -> Mapping[str, Any]: def stream_slices( self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None, **kwargs ) -> Iterable[Mapping[str, Any]]: - slices = [] # we can query a max of 100 contacts at a time diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py index 9b8378241dbfd..14a5737ab6c4f 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py @@ -85,7 +85,7 @@ def test_streams(requests_mock, config): streams = SourceHubspot().streams(config) - assert len(streams) == 29 + assert len(streams) == 30 def test_check_credential_title_exception(config): diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py index 518ceb5954c8a..bc7a150474cb0 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py @@ -28,6 +28,7 @@ LineItems, MarketingEmails, Owners, + OwnersArchived, Products, TicketPipelines, Tickets, @@ -107,6 +108,7 @@ def test_updated_at_field_non_exist_handler(requests_mock, common_params, fake_p (LineItems, "line_item", {"updatedAt": "2022-02-25T16:43:11Z"}), (MarketingEmails, "", {"updatedAt": "2022-02-25T16:43:11Z"}), (Owners, "", {"updatedAt": "2022-02-25T16:43:11Z"}), + (OwnersArchived, "", {"updatedAt": "2022-02-25T16:43:11Z"}), (Products, "product", {"updatedAt": "2022-02-25T16:43:11Z"}), (TicketPipelines, "", {"updatedAt": "2022-02-25T16:43:11Z"}), (Tickets, "ticket", {"updatedAt": "2022-02-25T16:43:11Z"}), diff --git a/docs/integrations/sources/hubspot.md b/docs/integrations/sources/hubspot.md index ba137bf7a2e4e..4781d56833817 100644 --- a/docs/integrations/sources/hubspot.md +++ b/docs/integrations/sources/hubspot.md @@ -204,6 +204,7 @@ Now that you have set up the Hubspot source connector, check out the following H | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1.4.0 | 2023-08-11 | [29249](https://github.com/airbytehq/airbyte/pull/29249) | Add `OwnersArchived` stream | | 1.3.3 | 2023-08-10 | [29248](https://github.com/airbytehq/airbyte/pull/29248) | Specify `threadId` in `engagements` stream to type string | | 1.3.2 | 2023-08-10 | [29326](https://github.com/airbytehq/airbyte/pull/29326) | Add primary keys to streams `ContactLists` and `PropertyHistory` | | 1.3.1 | 2023-08-08 | [29211](https://github.com/airbytehq/airbyte/pull/29211) | Handle 400 and 403 errors without interruption of the sync |