Migrate sendgrid to config-based

airbyte-integrations/connectors/source-sendgrid/source_sendgrid/sendgrid.yaml

@@ -0,0 +1,101 @@
+schema_loader:
+  type: JsonSchema
+  file_path: "./source_sendgrid/schemas/{{ options.name }}.json"
+result_selector:
+  type: RecordSelector
+  extractor:
+    type: JelloExtractor
+    transform: "_.result"
+requester:
+  type: HttpRequester
+  name: "{{ options['name'] }}"
+  url_base: "https://api.sendgrid.com"
+  http_method: "GET"
+  authenticator:
+    type: "BearerAuthenticator"
+    token: "{{ config.apikey }}"
+cursor_paginator:
+  type: LimitPaginator
+  url_base: "*ref(requester.url_base)"
+  page_size: 100
+  limit_option:
+    inject_into: "request_parameter"
+    field_name: "page_size"
+  page_token_option:
+    inject_into: "path"
+  pagination_strategy:
+    type: "CursorPagination"
+    cursor_value: "{{ response._metadata.next }}"
+offset_paginator:
+  type: LimitPaginator
+  $options:
+    url_base: "*ref(requester.url_base)"
+    page_size: 10
+  limit_option:
+    inject_into: "request_parameter"
+    field_name: "limit"
+  page_token_option:
+    inject_into: "request_parameter"
+    field_name: "offset"
+  pagination_strategy:
+    type: "OffsetIncrement"
+retriever:
+  type: SimpleRetriever
+  name: "{{ options['name'] }}"
+  primary_key: "{{ options['primary_key'] }}"
+
+lists_stream:
+  type: DeclarativeStream
+  $options:
+    name: "lists"
+  primary_key: "id"
+  schema_loader:
+    $ref: "*ref(schema_loader)"
+  retriever:
+    $ref: "*ref(retriever)"
+    record_selector:
+      $ref: "*ref(result_selector)"
+    requester:
+      $ref: "*ref(requester)"
+      path: "/v3/marketing/lists"
+    paginator:
+      $ref: "*ref(cursor_paginator)"
+campaigns_stream:
+  type: DeclarativeStream
+  $options:
+    name: "campaigns"
+  primary_key: "id"
+  schema_loader:
+    $ref: "*ref(schema_loader)"
+  retriever:
+    $ref: "*ref(retriever)"
+    record_selector:
+      $ref: "*ref(result_selector)"
+    requester:
+      $ref: "*ref(requester)"
+      path: "/v3/marketing/campaigns"
+    paginator:
+      $ref: "*ref(cursor_paginator)"
+contacts_stream:
+  type: DeclarativeStream
+  $options:
+    name: "contacts"
+  primary_key: "id"
+  schema_loader:
+    $ref: "*ref(schema_loader)"
+  retriever:
+    $ref: "*ref(retriever)"
+    record_selector:
+      $ref: "*ref(result_selector)"
+    requester:
+      $ref: "*ref(requester)"
+      path: "/v3/marketing/contacts"
+    paginator:
+      type: "NoPagination"
+streams:
+  - "*ref(lists_stream)"
+  - "*ref(campaigns_stream)"
+  - "*ref(contacts_stream)"
+check:
+  type: CheckStream
+  stream_names: ["lists"]

airbyte-integrations/connectors/source-sendgrid/source_sendgrid/source.py

@@ -2,63 +2,17 @@
 # Copyright (c) 2022 Airbyte, Inc., all rights reserved.
 #
 
+from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
 
-from typing import Any, List, Mapping, Tuple
+"""
+This file provides the necessary constructs to interpret a provided declarative YAML configuration file into
+source connector.
 
-from airbyte_cdk.models import SyncMode
-from airbyte_cdk.sources import AbstractSource
-from airbyte_cdk.sources.streams import Stream
-from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator
+WARNING: Do not modify this file.
+"""
 
-from .streams import (
-    Blocks,
-    Bounces,
-    Campaigns,
-    Contacts,
-    GlobalSuppressions,
-    InvalidEmails,
-    Lists,
-    Messages,
-    Scopes,
-    Segments,
-    SingleSends,
-    SpamReports,
-    StatsAutomations,
-    SuppressionGroupMembers,
-    SuppressionGroups,
-    Templates,
-)
 
-
-class SourceSendgrid(AbstractSource):
-    def check_connection(self, logger, config) -> Tuple[bool, any]:
-        try:
-            authenticator = TokenAuthenticator(config["apikey"])
-            scopes_gen = Scopes(authenticator=authenticator).read_records(sync_mode=SyncMode.full_refresh)
-            next(scopes_gen)
-            return True, None
-        except Exception as error:
-            return False, f"Unable to connect to Sendgrid API with the provided credentials - {error}"
-
-    def streams(self, config: Mapping[str, Any]) -> List[Stream]:
-        authenticator = TokenAuthenticator(config["apikey"])
-
-        streams = [
-            Lists(authenticator=authenticator),
-            Campaigns(authenticator=authenticator),
-            Contacts(authenticator=authenticator),
-            StatsAutomations(authenticator=authenticator),
-            Segments(authenticator=authenticator),
-            SingleSends(authenticator=authenticator),
-            Templates(authenticator=authenticator),
-            Messages(authenticator=authenticator, start_time=config["start_time"]),
-            GlobalSuppressions(authenticator=authenticator, start_time=config["start_time"]),
-            SuppressionGroups(authenticator=authenticator),
-            SuppressionGroupMembers(authenticator=authenticator),
-            Blocks(authenticator=authenticator, start_time=config["start_time"]),
-            Bounces(authenticator=authenticator, start_time=config["start_time"]),
-            InvalidEmails(authenticator=authenticator, start_time=config["start_time"]),
-            SpamReports(authenticator=authenticator, start_time=config["start_time"]),
-        ]
-
-        return streams
+# Declarative Source
+class SourceSendgrid(YamlDeclarativeSource):
+    def __init__(self):
+        super().__init__(**{"path_to_yaml": "./source_sendgrid/sendgrid.yaml"})

airbyte-integrations/connectors/source-sendgrid/source_sendgrid/spec.json

@@ -5,7 +5,7 @@
     "title": "Sendgrid Spec",
     "type": "object",
     "required": ["apikey"],
-    "additionalProperties": false,
+    "additionalProperties": true,
     "properties": {
       "apikey": {
         "title": "Sendgrid API key",