Source Hubspot: Add OwnersArchived stream (airbytehq#29249)

* Source Hubspot: Add OwnersArchived stream * Update changelog and CAT config * Update expected records, update schema * Update unit tests * Set archived to true explicitly
KYVENetwork · Sep 1, 2023 · b5f12c2 · b5f12c2
1 parent 1fa3886
commit b5f12c2
Show file tree

Hide file tree

Showing 10 changed files with 90 additions and 7 deletions.
diff --git a/airbyte-integrations/connectors/source-hubspot/Dockerfile b/airbyte-integrations/connectors/source-hubspot/Dockerfile
@@ -34,5 +34,5 @@ COPY source_hubspot ./source_hubspot
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=1.3.3
+LABEL io.airbyte.version=1.4.0
 LABEL io.airbyte.name=airbyte/source-hubspot
diff --git a/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml b/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml
@@ -41,6 +41,8 @@ acceptance_tests:
             bypass_reason: Unable to populate
           - name: deals_archived
             bypass_reason: Unable to populate
+          - name: owners_archived
+            bypass_reason: unable to populate
         ignored_fields:
           contact_lists:
             - name: ilsFilterBranch

diff --git a/airbyte-integrations/connectors/source-hubspot/metadata.yaml b/airbyte-integrations/connectors/source-hubspot/metadata.yaml
@@ -5,7 +5,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c
-  dockerImageTag: 1.3.3
+  dockerImageTag: 1.4.0
   dockerRepository: airbyte/source-hubspot
   githubIssueLabel: source-hubspot
   icon: hubspot.svg
@@ -14,7 +14,7 @@ data:
   registries:
     cloud:
       enabled: true
-      dockerImageTag: 1.3.3
+      dockerImageTag: 1.4.0
     oss:
       enabled: true
   releaseStage: generally_available

diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/basic_read_catalog.json b/airbyte-integrations/connectors/source-hubspot/sample_files/basic_read_catalog.json
@@ -222,6 +222,15 @@
       "sync_mode": "full_refresh",
       "destination_sync_mode": "overwrite"
     },
+    {
+      "stream": {
+        "name": "owners_archived",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
     {
       "stream": {
         "name": "products",

diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/owners_archived.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/owners_archived.json
@@ -0,0 +1,49 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": ["null", "object"],
+  "properties": {
+    "id": {
+      "type": ["null", "string"]
+    },
+    "email": {
+      "type": ["null", "string"]
+    },
+    "firstName": {
+      "type": ["null", "string"]
+    },
+    "lastName": {
+      "type": ["null", "string"]
+    },
+    "userId": {
+      "type": ["null", "integer"]
+    },
+    "createdAt": {
+      "type": ["null", "string"],
+      "format": "date-time"
+    },
+    "updatedAt": {
+      "type": ["null", "string"],
+      "format": "date-time"
+    },
+    "archived": {
+      "type": ["null", "boolean"]
+    },
+    "teams": {
+      "type": ["null", "array"],
+      "items": {
+        "type": "object",
+        "properties": {
+          "id": {
+            "type": ["null", "string"]
+          },
+          "name": {
+            "type": ["null", "string"]
+          },
+          "membership": {
+            "type": ["null", "string"]
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py
@@ -39,6 +39,7 @@
     LineItems,
     MarketingEmails,
     Owners,
+    OwnersArchived,
     Products,
     PropertyHistory,
     SubscriptionChanges,
@@ -120,6 +121,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
             LineItems(**common_params),
             MarketingEmails(**common_params),
             Owners(**common_params),
+            OwnersArchived(**common_params),
             Products(**common_params),
             PropertyHistory(**common_params),
             SubscriptionChanges(**common_params),

diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py
@@ -1552,7 +1552,6 @@ def read_records(
         stream_slice: Mapping[str, Any] = None,
         stream_state: Mapping[str, Any] = None,
     ) -> Iterable[Mapping[str, Any]]:
-
         self.latest_cursor = None
 
         # The date we need records since
@@ -1693,6 +1692,27 @@ class Owners(ClientSideIncrementalStream):
     scopes = {"crm.objects.owners.read"}
 
 
+class OwnersArchived(ClientSideIncrementalStream):
+    """Archived Owners, API v3"""
+
+    url = "/crm/v3/owners"
+    updated_at_field = "updatedAt"
+    created_at_field = "createdAt"
+    cursor_field_datetime_format = "YYYY-MM-DDTHH:mm:ss.SSSSSSZ"
+    primary_key = "id"
+    scopes = {"crm.objects.owners.read"}
+
+    def request_params(
+        self,
+        stream_state: Mapping[str, Any],
+        stream_slice: Mapping[str, Any] = None,
+        next_page_token: Mapping[str, Any] = None,
+    ) -> MutableMapping[str, Any]:
+        params = super().request_params(stream_state, stream_slice, next_page_token)
+        params["archived"] = "true"
+        return params
+
+
 class PropertyHistory(Stream):
     """Contacts Endpoint, API v1
     Is used to get all Contacts and the history of their respective
@@ -1790,7 +1810,6 @@ class Contacts(CRMSearchStream):
 
 
 class ContactsMergedAudit(Stream):
-
     url = "/contacts/v1/contact/vids/batch/"
     updated_at_field = "timestamp"
     scopes = {"crm.objects.contacts.read"}
@@ -1810,7 +1829,6 @@ def get_json_schema(self) -> Mapping[str, Any]:
     def stream_slices(
         self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None, **kwargs
     ) -> Iterable[Mapping[str, Any]]:
-
         slices = []
 
         # we can query a max of 100 contacts at a time

diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py
@@ -85,7 +85,7 @@ def test_streams(requests_mock, config):
 
     streams = SourceHubspot().streams(config)
 
-    assert len(streams) == 29
+    assert len(streams) == 30
 
 
 def test_check_credential_title_exception(config):

diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py
@@ -28,6 +28,7 @@
     LineItems,
     MarketingEmails,
     Owners,
+    OwnersArchived,
     Products,
     TicketPipelines,
     Tickets,
@@ -107,6 +108,7 @@ def test_updated_at_field_non_exist_handler(requests_mock, common_params, fake_p
         (LineItems, "line_item", {"updatedAt": "2022-02-25T16:43:11Z"}),
         (MarketingEmails, "", {"updatedAt": "2022-02-25T16:43:11Z"}),
         (Owners, "", {"updatedAt": "2022-02-25T16:43:11Z"}),
+        (OwnersArchived, "", {"updatedAt": "2022-02-25T16:43:11Z"}),
         (Products, "product", {"updatedAt": "2022-02-25T16:43:11Z"}),
         (TicketPipelines, "", {"updatedAt": "2022-02-25T16:43:11Z"}),
         (Tickets, "ticket", {"updatedAt": "2022-02-25T16:43:11Z"}),

diff --git a/docs/integrations/sources/hubspot.md b/docs/integrations/sources/hubspot.md
@@ -204,6 +204,7 @@ Now that you have set up the Hubspot source connector, check out the following H
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                                                                                            |
 | :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| 1.4.0   | 2023-08-11 | [29249](https://github.com/airbytehq/airbyte/pull/29249) | Add `OwnersArchived` stream                                                                                                                                               |
 | 1.3.3   | 2023-08-10 | [29248](https://github.com/airbytehq/airbyte/pull/29248) | Specify `threadId` in `engagements` stream to type string                                                                                                                                               |
 | 1.3.2   | 2023-08-10 | [29326](https://github.com/airbytehq/airbyte/pull/29326) | Add primary keys to streams `ContactLists` and `PropertyHistory` |
 | 1.3.1   | 2023-08-08 | [29211](https://github.com/airbytehq/airbyte/pull/29211) | Handle 400 and 403 errors without interruption of the sync                                |