From fae4d00c8fa6454964ba01b6b09f80bf0665cf57 Mon Sep 17 00:00:00 2001 From: Hai To Date: Thu, 21 Oct 2021 01:37:48 +0200 Subject: [PATCH] Source Hubspot: Add new marketing emails (with statistics) stream (#5840) * Add new marketing emails (with statistics) stream * Update docs and changelog --- .../connectors/source-hubspot/Dockerfile | 2 +- .../sample_files/configured_catalog.json | 12 + .../configured_catalog_for_oauth_config.json | 12 + .../source-hubspot/source_hubspot/api.py | 12 + .../source-hubspot/source_hubspot/client.py | 2 + .../schemas/marketing_emails.json | 522 ++++++++++++++++++ docs/integrations/sources/hubspot.md | 3 +- 7 files changed, 563 insertions(+), 2 deletions(-) create mode 100644 airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/marketing_emails.json diff --git a/airbyte-integrations/connectors/source-hubspot/Dockerfile b/airbyte-integrations/connectors/source-hubspot/Dockerfile index 90896aec09d2a..fffe2d113da27 100644 --- a/airbyte-integrations/connectors/source-hubspot/Dockerfile +++ b/airbyte-integrations/connectors/source-hubspot/Dockerfile @@ -14,5 +14,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "/airbyte/base.sh" -LABEL io.airbyte.version=0.1.17 +LABEL io.airbyte.version=0.1.18 LABEL io.airbyte.name=airbyte/source-hubspot diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog.json index 41ab4edb47300..3eb0dc3836cae 100644 --- a/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog.json @@ -93,6 +93,18 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, + { + "stream": { + "name": "marketing_emails", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false, + "default_cursor_field": ["updated"] + }, + "sync_mode": "full_refresh", + "cursor_field": null, + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "owners", diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog_for_oauth_config.json b/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog_for_oauth_config.json index 368e0b73064a5..ee85be89faf26 100644 --- a/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog_for_oauth_config.json +++ b/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog_for_oauth_config.json @@ -93,6 +93,18 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, + { + "stream": { + "name": "marketing_emails", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false, + "default_cursor_field": ["updated"] + }, + "sync_mode": "full_refresh", + "cursor_field": null, + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "owners", diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py index 071f103aaff2f..a5d7f58dac409 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py @@ -650,6 +650,18 @@ class FormStream(Stream): created_at_field = "createdAt" +class MarketingEmailStream(Stream): + """Marketing Email, API v1 + Docs: https://legacydocs.hubspot.com/docs/methods/cms_email/get-all-marketing-emails + """ + + url = "/marketing-emails/v1/emails/with-statistics" + data_field = "objects" + limit = 250 + updated_at_field = "updated" + created_at_field = "created" + + class OwnerStream(Stream): """Owners, API v3 Docs: https://legacydocs.hubspot.com/docs/methods/owners/get_owners diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/client.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/client.py index 58430e57ffb20..6f6a2cf865ffe 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/client.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/client.py @@ -18,6 +18,7 @@ EmailEventStream, EngagementStream, FormStream, + MarketingEmailStream, OwnerStream, SubscriptionChangeStream, WorkflowStream, @@ -43,6 +44,7 @@ def __init__(self, start_date, credentials, **kwargs): "engagements": EngagementStream(**common_params), "forms": FormStream(**common_params), "line_items": CRMObjectStream(entity="line_item", **common_params), + "marketing_emails": MarketingEmailStream(**common_params), "owners": OwnerStream(**common_params), "products": CRMObjectStream(entity="product", **common_params), "subscription_changes": SubscriptionChangeStream(**common_params), diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/marketing_emails.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/marketing_emails.json new file mode 100644 index 0000000000000..29cff48bfc5ba --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/marketing_emails.json @@ -0,0 +1,522 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "ab": { + "type": ["null", "boolean"] + }, + "abHoursToWait": { + "type": ["null", "integer"] + }, + "abSampleSizeDefault": { + "type": ["null", "string"] + }, + "abSamplingDefault": { + "type": ["null", "string"] + }, + "abStatus": { + "type": ["null", "string"] + }, + "abSuccessMetric": { + "type": ["null", "string"] + }, + "abTestPercentage": { + "type": ["null", "integer"] + }, + "abVariation": { + "type": ["null", "boolean"] + }, + "absoluteUrl": { + "type": ["null", "string"] + }, + "allEmailCampaignIds": { + "type": ["null", "array"], + "items": { + "type": [ + "null", + "integer" + ] + } + }, + "analyticsPageId": { + "type": ["null", "string"] + }, + "analyticsPageType": { + "type": ["null", "string"] + }, + "archived": { + "type": ["null", "boolean"] + }, + "authorAt": { + "type": ["null", "integer"] + }, + "authorName": { + "type": ["null", "string"] + }, + "authorUserId": { + "type": ["null", "integer"] + }, + "blogEmailType": { + "type": ["null", "string"] + }, + "campaign": { + "type": ["null", "string"] + }, + "campaignName": { + "type": ["null", "string"] + }, + "canSpamSettingsId": { + "type": ["null", "integer"] + }, + "categoryId": { + "type": ["null", "integer"] + }, + "clonedFrom": { + "type": ["null", "integer"] + }, + "contentTypeCategory": { + "type": ["null", "integer"] + }, + "createPage": { + "type": ["null", "boolean"] + }, + "created": { + "type": ["null", "integer"] + }, + "createdById": { + "type": ["null", "integer"] + }, + "currentState": { + "type": ["null", "string"] + }, + "currentlyPublished": { + "type": ["null", "boolean"] + }, + "customReplyTo": { + "type": ["null", "string"] + }, + "customReplyToEnabled": { + "type": ["null", "boolean"] + }, + "domain": { + "type": ["null", "string"] + }, + "emailBody": { + "type": ["null", "string"] + }, + "emailNote": { + "type": ["null", "string"] + }, + "emailTemplateMode": { + "type": ["null", "string"] + }, + "emailType": { + "type": ["null", "string"] + }, + "emailbodyPlaintext": { + "type": ["null", "string"] + }, + "feedbackEmailCategory": { + "type": ["null", "string"] + }, + "feedbackSurveyId": { + "type": ["null", "integer"] + }, + "folderId": { + "type": ["null", "integer"] + }, + "freezeDate": { + "type": ["null", "integer"] + }, + "fromName": { + "type": ["null", "string"] + }, + "htmlTitle": { + "type": ["null", "string"] + }, + "isGraymailSuppressionEnabled": { + "type": ["null", "boolean"] + }, + "isLocalTimezoneSend": { + "type": ["null", "boolean"] + }, + "isPublished": { + "type": ["null", "boolean"] + }, + "isRecipientFatigueSuppressionEnabled": { + "type": ["null", "boolean"] + }, + "leadFlowId": { + "type": ["null", "integer"] + }, + "liveDomain": { + "type": ["null", "string"] + }, + "mailingListsExcluded": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "mailingListsIncluded": { + "type": ["null", "string"], + "items": { + "type": ["null", "integer"] + } + }, + "maxRssEntries": { + "type": ["null", "integer"] + }, + "metaDescription": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "pageExpiryEnabled": { + "type": ["null", "boolean"] + }, + "pageRedirected": { + "type": ["null", "boolean"] + }, + "portalId": { + "type": ["null", "integer"] + }, + "previewKey": { + "type": ["null", "string"] + }, + "primaryEmailCampaignId": { + "type": ["null", "string"] + }, + "processingStatus": { + "type": ["null", "string"] + }, + "publishDate": { + "type": ["null", "integer"] + }, + "publishedById": { + "type": ["null", "integer"] + }, + "publishedByName": { + "type": ["null", "string"] + }, + "publishImmediately": { + "type": ["null", "boolean"] + }, + "publishedUrl": { + "type": ["null", "string"] + }, + "replyTo": { + "type": ["null", "string"] + }, + "resolvedDomain": { + "type": ["null", "string"] + }, + "selected": { + "type": ["null", "integer"] + }, + "slug": { + "type": ["null", "string"] + }, + "smartEmailFields": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "stats": { + "type": [ + "null", + "object" + ], + "properties": { + "counters": { + "type": [ + "null", + "object" + ], + "properties": { + "open": { + "type": [ + "null", + "integer" + ] + }, + "delivered": { + "type": [ + "null", + "integer" + ] + }, + "bounce": { + "type": [ + "null", + "integer" + ] + }, + "unsubscribed": { + "type": [ + "null", + "integer" + ] + }, + "click": { + "type": [ + "null", + "integer" + ] + }, + "dropped": { + "type": [ + "null", + "integer" + ] + }, + "selected": { + "type": [ + "null", + "integer" + ] + }, + "spamreport": { + "type": [ + "null", + "integer" + ] + }, + "suppressed": { + "type": [ + "null", + "integer" + ] + }, + "hardbounced": { + "type": [ + "null", + "integer" + ] + }, + "softbounced": { + "type": [ + "null", + "integer" + ] + }, + "pending": { + "type": [ + "null", + "integer" + ] + }, + "contactslost": { + "type": [ + "null", + "integer" + ] + }, + "notsent": { + "type": [ + "null", + "integer" + ] + } + } + }, + "deviceBreakdown": { + "type": [ + "null", + "object" + ], + "properties": { + "open_device_type": { + "type": [ + "null", + "object" + ], + "properties": { + "computer": { + "type": [ + "null", + "integer" + ] + }, + "mobile": { + "type": [ + "null", + "integer" + ] + }, + "unknown": { + "type": [ + "null", + "integer" + ] + } + } + }, + "click_device_type": { + "type": [ + "null", + "object" + ], + "properties": { + "computer": { + "type": [ + "null", + "integer" + ] + }, + "mobile": { + "type": [ + "null", + "integer" + ] + }, + "unknown": { + "type": [ + "null", + "integer" + ] + } + } + } + } + }, + "failedToLoad": { + "type": [ + "null", + "boolean" + ] + }, + "ratios": { + "type": [ + "null", + "object" + ], + "properties": { + "clickratio": { + "type": [ + "null", + "number" + ] + }, + "clickthroughratio": { + "type": [ + "null", + "number" + ] + }, + "deliveredratio": { + "type": [ + "null", + "number" + ] + }, + "openratio": { + "type": [ + "null", + "number" + ] + }, + "unsubscribedratio": { + "type": [ + "null", + "number" + ] + }, + "spamreportratio": { + "type": [ + "null", + "number" + ] + }, + "bounceratio": { + "type": [ + "null", + "number" + ] + }, + "hardbounceratio": { + "type": [ + "null", + "number" + ] + }, + "softbounceratio": { + "type": [ + "null", + "number" + ] + }, + "contactslostratio": { + "type": [ + "null", + "number" + ] + }, + "pendingratio": { + "type": [ + "null", + "number" + ] + }, + "notsentratio": { + "type": [ + "null", + "number" + ] + } + } + } + } + }, + "subcategory": { + "type": ["null", "string"] + }, + "subject": { + "type": ["null", "string"] + }, + "subscription": { + "type": ["null", "integer"] + }, + "subscriptionName": { + "type": ["null", "string"] + }, + "templatePath": { + "type": ["null", "string"] + }, + "transactional": { + "type": ["null", "boolean"] + }, + "unpublishedAt": { + "type": ["null", "integer"] + }, + "updated": { + "type": ["null", "integer"] + }, + "updatedById": { + "type": ["null", "integer"] + }, + "url": { + "type": ["null", "string"] + }, + "useRssHeadlineAsSubject": { + "type": ["null", "boolean"] + }, + "vidsExcluded": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "vidsIncluded": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + } + } +} diff --git a/docs/integrations/sources/hubspot.md b/docs/integrations/sources/hubspot.md index 5c5242545076a..78cc912b1f84f 100644 --- a/docs/integrations/sources/hubspot.md +++ b/docs/integrations/sources/hubspot.md @@ -27,6 +27,7 @@ This source is capable of syncing the following tables and their data: * [Engagements](https://legacydocs.hubspot.com/docs/methods/engagements/get-all-engagements) * [Forms](https://developers.hubspot.com/docs/api/marketing/forms) * [Line Items](https://developers.hubspot.com/docs/api/crm/line-items) +* [Marketing Emails](https://legacydocs.hubspot.com/docs/methods/cms_email/get-all-marketing-email-statistics) * [Owners](https://developers.hubspot.com/docs/methods/owners/get_owners) * [Products](https://developers.hubspot.com/docs/api/crm/products) * [Quotes](https://developers.hubspot.com/docs/api/crm/quotes) @@ -95,6 +96,7 @@ If you are using Oauth, most of the streams require the appropriate [scopes](htt | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.18 | 2021-10-18 | [5840](https://github.com/airbytehq/airbyte/pull/5840) | Add new marketing emails (with statistics) stream | | 0.1.17 | 2021-10-14 | [6995](https://github.com/airbytehq/airbyte/pull/6995) | Update `discover` method: disable `quotes` stream when using OAuth config | | 0.1.16 | 2021-09-27 | [6465](https://github.com/airbytehq/airbyte/pull/6465) | Implement OAuth support. Use CDK authenticator instead of connector specific authenticator | | 0.1.15 | 2021-09-23 | [6374](https://github.com/airbytehq/airbyte/pull/6374) | Use correct schema for `owners` stream | @@ -106,4 +108,3 @@ If you are using Oauth, most of the streams require the appropriate [scopes](htt | 0.1.9 | 2021-08-11 | [5334](https://github.com/airbytehq/airbyte/pull/5334) | Fix empty strings inside float datatype | | 0.1.8 | 2021-08-06 | [5250](https://github.com/airbytehq/airbyte/pull/5250) | Fix issue with printing exceptions | | 0.1.7 | 2021-07-27 | [4913](https://github.com/airbytehq/airbyte/pull/4913) | Update fields schema | -