Source Sendgrid: change start time param type to datetime string

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -908,7 +908,7 @@
 - name: Sendgrid
   sourceDefinitionId: fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87
   dockerRepository: airbyte/source-sendgrid
-  dockerImageTag: 0.2.13
+  dockerImageTag: 0.2.14
   documentationUrl: https://docs.airbyte.io/integrations/sources/sendgrid
   icon: sendgrid.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -9365,7 +9365,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-sendgrid:0.2.13"
+- dockerImage: "airbyte/source-sendgrid:0.2.14"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/sendgrid"
     connectionSpecification:
@@ -9384,11 +9384,16 @@
           order: 0
         start_time:
           title: "Start time"
-          type: "integer"
-          description: "Start time in timestamp integer format. Any data before this\
-            \ timestamp will not be replicated."
+          type:
+          - "integer"
+          - "string"
+          description: "Start time in ISO8601 format. Any data before this time point\
+            \ will not be replicated."
           examples:
-          - 1558359837
+          - "2021-12-12"
+          - "2021-02-01 13:30:00"
+          - "2020-07-18T13:30:00.000Z"
+          - "2020-07-18 13:30:00+02:00"
           order: 1
     supportsNormalization: false
     supportsDBT: false

airbyte-integrations/bases/source-acceptance-test/unit_tests/test_backward_compatibility.py

@@ -1167,12 +1167,14 @@ def test_validate_previous_configs(previous_connector_spec, actual_connector_spe
             "test_stream": AirbyteStream.parse_obj(
                 {
                     "name": "test_stream",
-                    "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": ["integer", "string"]}}}}},
+                    "json_schema": {
+                        "properties": {"user": {"type": "object", "properties": {"username": {"type": ["integer", "string"]}}}}
+                    },
                     "default_cursor_field": ["b"],
                 }
             ),
         },
-    )
+    ),
 ]
 
 VALID_CATALOG_TRANSITIONS = [

airbyte-integrations/connectors/source-sendgrid/Dockerfile

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.13
+LABEL io.airbyte.version=0.2.14
 LABEL io.airbyte.name=airbyte/source-sendgrid

airbyte-integrations/connectors/source-sendgrid/acceptance-test-config.yml

@@ -5,10 +5,14 @@ tests:
   connection:
     - config_path: "secrets/config.json"
       status: "succeed"
-    - config_path: "integration_tests/invalid_config.json"
+    - config_path: "secrets/old_config.json"
+      status: "succeed"
+    - config_path: "integration_tests/invalid_time.json"
+      status: "failed"
+    - config_path: "integration_tests/invalid_api_key.json"
       status: "failed"
   discovery:
-    - config_path: "secrets/config.json"
+    - config_path: "secrets/old_config.json"
   basic_read:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/no_spam_reports_configured_catalog.json"

airbyte-integrations/connectors/source-sendgrid/integration_tests/invalid_time.json

@@ -0,0 +1,4 @@
+{
+  "apikey": "apikey",
+  "start_time": "some erroneous input"
+}

airbyte-integrations/connectors/source-sendgrid/source_sendgrid/source.py

@@ -5,6 +5,7 @@
 
 from typing import Any, List, Mapping, Tuple
 
+import pendulum
 from airbyte_cdk.models import SyncMode
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.streams import Stream
@@ -33,15 +34,21 @@
 class SourceSendgrid(AbstractSource):
     def check_connection(self, logger, config) -> Tuple[bool, any]:
         try:
+            start_time = config.get("start_time")
+            if start_time and isinstance(start_time, str):
+                pendulum.parse(start_time)
             authenticator = TokenAuthenticator(config["apikey"])
             scopes_gen = Scopes(authenticator=authenticator).read_records(sync_mode=SyncMode.full_refresh)
             next(scopes_gen)
             return True, None
+        except pendulum.parsing.exceptions.ParserError:
+            return False, "Please, provide a valid Start Time parameter"
         except Exception as error:
             return False, f"Unable to connect to Sendgrid API with the provided credentials - {error}"
 
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
         authenticator = TokenAuthenticator(config["apikey"])
+        start_time = config.get("start_time")
 
         streams = [
             Lists(authenticator=authenticator),
@@ -51,14 +58,14 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
             Segments(authenticator=authenticator),
             SingleSends(authenticator=authenticator),
             Templates(authenticator=authenticator),
-            Messages(authenticator=authenticator, start_time=config["start_time"]),
-            GlobalSuppressions(authenticator=authenticator, start_time=config["start_time"]),
+            Messages(authenticator=authenticator, start_time=start_time),
+            GlobalSuppressions(authenticator=authenticator, start_time=start_time),
             SuppressionGroups(authenticator=authenticator),
             SuppressionGroupMembers(authenticator=authenticator),
-            Blocks(authenticator=authenticator, start_time=config["start_time"]),
-            Bounces(authenticator=authenticator, start_time=config["start_time"]),
-            InvalidEmails(authenticator=authenticator, start_time=config["start_time"]),
-            SpamReports(authenticator=authenticator, start_time=config["start_time"]),
+            Blocks(authenticator=authenticator, start_time=start_time),
+            Bounces(authenticator=authenticator, start_time=start_time),
+            InvalidEmails(authenticator=authenticator, start_time=start_time),
+            SpamReports(authenticator=authenticator, start_time=start_time),
         ]
 
         return streams

airbyte-integrations/connectors/source-sendgrid/source_sendgrid/spec.json

@@ -15,9 +15,9 @@
       },
       "start_time": {
         "title": "Start time",
-        "type": "integer",
-        "description": "Start time in timestamp integer format. Any data before this timestamp will not be replicated.",
-        "examples": [1558359837],
+        "type": ["integer", "string"],
+        "description": "Start time in ISO8601 format. Any data before this time point will not be replicated.",
+        "examples": ["2021-12-12", "2021-02-01 13:30:00", "2020-07-18T13:30:00.000Z", "2020-07-18 13:30:00+02:00"],
         "order": 1
       }
     }

airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py

@@ -5,7 +5,7 @@
 import datetime
 import urllib
 from abc import ABC, abstractmethod
-from typing import Any, Iterable, Mapping, MutableMapping, Optional
+from typing import Any, Iterable, Mapping, MutableMapping, Optional, Union
 
 import pendulum
 import requests
@@ -72,9 +72,13 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str,
 class SendgridStreamIncrementalMixin(HttpStream, ABC):
     cursor_field = "created"
 
-    def __init__(self, start_time: int, **kwargs):
+    def __init__(self, start_time: Optional[Union[int, str]], **kwargs):
         super().__init__(**kwargs)
+        self._start_time = start_time or 0
+        # for backward compatibility
         self._start_time = start_time
+        if isinstance(self._start_time, str):
+            self._start_time = int(pendulum.parse(self._start_time).timestamp())
 
     def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]:
         """

docs/integrations/sources/sendgrid.md

@@ -76,13 +76,14 @@ The connector is restricted by normal Sendgrid [requests limitation](https://sen
 
 ## Changelog
 
-| Version | Date       | Pull Request                                             | Subject                                            |
-|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------|
-| 0.2.13  | 2022-08-29 | [16112](https://github.com/airbytehq/airbyte/pull/16112) | Revert back to Python CDK                          |
-| 0.2.12  | 2022-08-24 | [15911](https://github.com/airbytehq/airbyte/pull/15911) | Bugfix to allowing reading schemas at runtime      |
-| 0.2.11  | 2022-08-19 | [15800](https://github.com/airbytehq/airbyte/pull/15800) | Bugfix to allow reading sentry.yaml at runtime     |
-| 0.2.10  | 2022-08-17 | [15734](https://github.com/airbytehq/airbyte/pull/15734) | Fix yaml based on the new schema validator         |
-| 0.2.9   | 2022-08-11 | [15257](https://github.com/airbytehq/airbyte/pull/15257) | Migrate to config-based framework                  |
-| 0.2.8   | 2022-06-07 | [13571](https://github.com/airbytehq/airbyte/pull/13571) | Add Message stream                                 |
-| 0.2.7   | 2021-09-08 | [5910](https://github.com/airbytehq/airbyte/pull/5910)   | Add Single Sends Stats stream                      |
-| 0.2.6   | 2021-07-19 | [4839](https://github.com/airbytehq/airbyte/pull/4839)   | Gracefully handle malformed responses from the API |
+| Version | Date       | Pull Request                                             | Subject                                               |
+|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------|
+| 0.2.14  | 2022-09-07 | [16400](https://github.com/airbytehq/airbyte/pull/16400) | Change Start Time config parameter to datetime string |
+| 0.2.13  | 2022-08-29 | [16112](https://github.com/airbytehq/airbyte/pull/16112) | Revert back to Python CDK                             |
+| 0.2.12  | 2022-08-24 | [15911](https://github.com/airbytehq/airbyte/pull/15911) | Bugfix to allowing reading schemas at runtime         |
+| 0.2.11  | 2022-08-19 | [15800](https://github.com/airbytehq/airbyte/pull/15800) | Bugfix to allow reading sentry.yaml at runtime        |
+| 0.2.10  | 2022-08-17 | [15734](https://github.com/airbytehq/airbyte/pull/15734) | Fix yaml based on the new schema validator            |
+| 0.2.9   | 2022-08-11 | [15257](https://github.com/airbytehq/airbyte/pull/15257) | Migrate to config-based framework                     |
+| 0.2.8   | 2022-06-07 | [13571](https://github.com/airbytehq/airbyte/pull/13571) | Add Message stream                                    |
+| 0.2.7   | 2021-09-08 | [5910](https://github.com/airbytehq/airbyte/pull/5910)   | Add Single Sends Stats stream                         |
+| 0.2.6   | 2021-07-19 | [4839](https://github.com/airbytehq/airbyte/pull/4839)   | Gracefully handle malformed responses from the API    |