🚨🚨🐛 Source Mailchimp: Multiple schema updates and fixes

airbyte-integrations/connectors/source-mailchimp/metadata.yaml

@@ -10,7 +10,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: b03a9f3e-22a5-11eb-adc1-0242ac120002
-  dockerImageTag: 0.10.0
+  dockerImageTag: 0.10.1
   dockerRepository: airbyte/source-mailchimp
   documentationUrl: https://docs.airbyte.com/integrations/sources/mailchimp
   githubIssueLabel: source-mailchimp
@@ -23,6 +23,12 @@ data:
     oss:
       enabled: true
   releaseStage: generally_available
+  suggestedStreams:
+    streams:
+      - email_activity
+      - campaigns
+      - lists
+      - reports
   supportLevel: certified
   tags:
     - language:python

airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/automations.json

@@ -8,10 +8,12 @@
       "type": ["null", "string"]
     },
     "create_time": {
-      "type": ["null", "string"]
+      "type": ["null", "string"],
+      "format": "date-time"
     },
     "start_time": {
-      "type": ["null", "string"]
+      "type": ["null", "string"],
+      "format": "date-time"
     },
     "status": {
       "type": ["null", "string"]

airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/campaigns.json

@@ -28,7 +28,8 @@
       "type": "string",
       "title": "Create Time",
       "description": "The date and time the campaign was created in ISO 8601 format.",
-      "readOnly": true
+      "readOnly": true,
+      "format": "date-time"
     },
     "archive_url": {
       "type": "string",
@@ -52,10 +53,11 @@
       "readOnly": true
     },
     "send_time": {
-      "type": "string",
+      "type": ["null", "string"],
       "title": "Send Time",
       "description": "The date and time a campaign was sent.",
-      "readOnly": true
+      "readOnly": true,
+      "format": "date-time"
     },
     "content_type": {
       "type": "string",
@@ -258,7 +260,8 @@
           "title": "Send Times",
           "description": "The possible send times to test. The times provided should be in the format YYYY-MM-DD HH:MM:SS. If send_times are provided to test, the test_size will be set to 100% and winner_criteria will be ignored.",
           "items": {
-            "type": "string"
+            "type": ["null", "string"],
+            "format": "date-time"
           }
         },
         "from_names": {
@@ -503,7 +506,8 @@
           "type": "string",
           "title": "Last Sent",
           "description": "The date the campaign was last sent.",
-          "readOnly": true
+          "readOnly": true,
+          "format": "date-time"
         },
         "constrain_rss_img": {
           "type": "boolean",
@@ -579,12 +583,14 @@
           "description": "For campaigns split on 'Subject Line', the subject line for Group B."
         },
         "send_time_a": {
-          "type": "string",
+          "type": ["null", "string"],
+          "format": "date-time",
           "title": "Send Time Group A",
           "description": "The send time for Group A."
         },
         "send_time_b": {
-          "type": "string",
+          "type": ["null", "string"],
+          "format": "date-time",
           "title": "Send Time Group B",
           "description": "The send time for Group B."
         },

airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/list_members.json

@@ -76,13 +76,15 @@
       "type": ["null", "string"]
     },
     "timestamp_opt": {
-      "type": ["null", "string"]
+      "type": ["null", "string"],
+      "format": "date-time"
     },
     "member_rating": {
       "type": ["null", "integer"]
     },
     "last_changed": {
-      "type": ["null", "string"]
+      "type": ["null", "string"],
+      "format": "date-time"
     },
     "language": {
       "type": ["null", "string"]

airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/lists.json

@@ -245,7 +245,8 @@
           "type": "string",
           "title": "Campaign Last Sent",
           "description": "The date and time the last campaign was sent to this list in ISO 8601 format. This is updated when a campaign is sent to 10 or more recipients.",
-          "readOnly": true
+          "readOnly": true,
+          "format": "date-time"
         },
         "merge_field_count": {
           "type": "integer",
@@ -287,13 +288,15 @@
           "type": "string",
           "title": "Date of Last List Subscribe",
           "description": "The date and time of the last time someone subscribed to this list in ISO 8601 format.",
-          "readOnly": true
+          "readOnly": true,
+          "format": "date-time"
         },
         "last_unsub_date": {
           "type": "string",
           "title": "Date of Last List Unsubscribe",
           "description": "The date and time of the last time someone unsubscribed from this list in ISO 8601 format.",
-          "readOnly": true
+          "readOnly": true,
+          "format": "date-time"
         }
       }
     }

airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/reports.json

@@ -321,7 +321,8 @@
             "last_open": {
               "type": "string",
               "title": "Last Open",
-              "description": "The last open for Campaign A."
+              "description": "The last open for Campaign A.",
+              "format": "date-time"
             },
             "unique_opens": {
               "type": "integer",
@@ -373,7 +374,8 @@
             "last_open": {
               "type": "string",
               "title": "Last Open",
-              "description": "The last open for Campaign B."
+              "description": "The last open for Campaign B.",
+              "format": "date-time"
             },
             "unique_opens": {
               "type": "integer",

airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/segment_members.json

@@ -48,7 +48,8 @@
       "type": ["null", "string"]
     },
     "timestamp_opt": {
-      "type": ["null", "string"]
+      "type": ["null", "string"],
+      "format": "date-time"
     },
     "member_rating": {
       "type": ["null", "integer"]

airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/unsubscribes.json

@@ -17,7 +17,8 @@
       "type": ["null", "boolean"]
     },
     "timestamp": {
-      "type": ["null", "string"]
+      "type": ["null", "string"],
+      "format": "date-time"
     },
     "reason": {
       "type": ["null", "string"]

airbyte-integrations/connectors/source-mailchimp/source_mailchimp/streams.py

@@ -45,7 +45,8 @@ def request_params(
         next_page_token: Mapping[str, Any] = None,
     ) -> MutableMapping[str, Any]:
 
-        params = {"count": self.page_size}
+        # The ._links field is returned by most Mailchimp endpoints and contains non-relevant schema metadata.
+        params = {"count": self.page_size, "exclude_fields": f"{self.data_field}._links"}
 
         # Handle pagination by inserting the next page's token in the request parameters
         if next_page_token:
@@ -143,9 +144,6 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
     def request_params(self, stream_state=None, stream_slice=None, **kwargs) -> MutableMapping[str, Any]:
         params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, **kwargs)
 
-        # Exclude the _links field, as it is not user-relevant data
-        params["exclude_fields"] = f"{self.data_field}._links"
-
         # Get the current state value for this list_id, if it exists
         # Then, use the value in state to filter the request
         current_slice = stream_slice.get("list_id")
@@ -185,6 +183,14 @@ class Campaigns(IncrementalMailChimpStream):
     def path(self, **kwargs) -> str:
         return "campaigns"
 
+    def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
+        response = super().parse_response(response, **kwargs)
+        for record in response:
+            # Set the send_time to None if it is an empty string to avoid validation errors
+            if record.get("send_time") == "":
+                record["send_time"] = None
+            yield record
+
 
 class Automations(IncrementalMailChimpStream):
     """Doc Link: https://mailchimp.com/developer/marketing/api/automation/get-automation-info/"""
@@ -275,13 +281,6 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
         list_id = stream_slice.get("parent").get("id")
         return f"lists/{list_id}/interest-categories"
 
-    def request_params(self, **kwargs):
-
-        # Exclude the _links field, as it is not user-relevant data
-        params = super().request_params(**kwargs)
-        params["exclude_fields"] = "categories._links"
-        return params
-
 
 class Interests(MailChimpStream, HttpSubStream):
     """
@@ -299,13 +298,6 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
         category_id = stream_slice.get("parent").get("id")
         return f"lists/{list_id}/interest-categories/{category_id}/interests"
 
-    def request_params(self, **kwargs):
-
-        # Exclude the _links field, as it is not user-relevant data
-        params = super().request_params(**kwargs)
-        params["exclude_fields"] = "interests._links"
-        return params
-
 
 class ListMembers(MailChimpListSubStream):
     """
@@ -485,12 +477,6 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
         campaign_id = stream_slice.get("campaign_id")
         return f"reports/{campaign_id}/unsubscribed"
 
-    def request_params(self, stream_state=None, stream_slice=None, **kwargs) -> MutableMapping[str, Any]:
-        params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, **kwargs)
-        # Exclude the _links field, as it is not user-relevant data
-        params["exclude_fields"] = "unsubscribes._links"
-        return params
-
     def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]:
 
         response = super().parse_response(response, **kwargs)

airbyte-integrations/connectors/source-mailchimp/unit_tests/test_streams.py

@@ -76,12 +76,12 @@ def test_next_page_token(auth):
         (
             Lists,
             {"stream_slice": None, "stream_state": None, "next_page_token": None},
-            {"count": 1000, "sort_dir": "ASC", "sort_field": "date_created"},
+            {"count": 1000, "sort_dir": "ASC", "sort_field": "date_created", "exclude_fields": "lists._links"},
         ),
         (
             Lists,
             {"stream_slice": None, "stream_state": None, "next_page_token": {"offset": 1000}},
-            {"count": 1000, "sort_dir": "ASC", "sort_field": "date_created", "offset": 1000},
+            {"count": 1000, "sort_dir": "ASC", "sort_field": "date_created", "offset": 1000, "exclude_fields": "lists._links"},
         ),
         (
             InterestCategories,