Migrate sendgrid to config-based

airbyte-integrations/connectors/source-sendgrid/setup.py

@@ -11,6 +11,6 @@
     author="Airbyte",
     author_email="contact@airbyte.io",
     packages=find_packages(),
-    install_requires=["airbyte-cdk~=0.1", "backoff", "requests", "pytest==6.1.2", "pytest-mock"],
+    install_requires=["airbyte-cdk>=0.1.74", "backoff", "requests", "pytest==6.1.2", "pytest-mock"],
     package_data={"": ["*.json", "schemas/*.json"]},
 )

airbyte-integrations/connectors/source-sendgrid/source_sendgrid/sendgrid.yaml

@@ -0,0 +1,342 @@
+definitions:
+  page_size: 50
+
+  schema_loader:
+    type: JsonSchema
+    file_path: "./source_sendgrid/schemas/{{ options.name }}.json"
+  result_selector:
+    type: RecordSelector
+    extractor:
+      type: DpathExtractor
+      field_pointer:
+        - "result"
+  results_selector:
+    type: RecordSelector
+    extractor:
+      type: DpathExtractor
+      field_pointer:
+        - "results"
+  selector:
+    type: RecordSelector
+    extractor:
+      type: DpathExtractor
+      field_pointer: []
+  requester:
+    type: HttpRequester
+    name: "{{ options['name'] }}"
+    url_base: "https://api.sendgrid.com"
+    http_method: "GET"
+    authenticator:
+      type: "BearerAuthenticator"
+      api_token: "{{ config.apikey }}"
+  cursor_paginator:
+    type: LimitPaginator
+    url_base: "*ref(definitions.requester.url_base)"
+    page_size: "*ref(definitions.page_size)"
+    limit_option:
+      inject_into: "request_parameter"
+      field_name: "page_size"
+    page_token_option:
+      inject_into: "path"
+    pagination_strategy:
+      type: "CursorPagination"
+      cursor_value: "{{ response._metadata.next }}"
+  offset_paginator:
+    type: LimitPaginator
+    $options:
+      url_base: "*ref(definitions.requester.url_base)"
+      page_size: "*ref(definitions.page_size)"
+    limit_option:
+      inject_into: "request_parameter"
+      field_name: "limit"
+    page_token_option:
+      inject_into: "request_parameter"
+      field_name: "offset"
+    pagination_strategy:
+      type: "OffsetIncrement"
+  retriever:
+    type: SimpleRetriever
+    name: "{{ options['name'] }}"
+    primary_key: "{{ options['primary_key'] }}"
+  stream_slicer:
+    type: "DatetimeStreamSlicer"
+    start_datetime:
+      datetime: "2017-01-01T00:00:00.0Z"
+      datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z"
+    end_datetime:
+      datetime: "2022-08-01T00:00:00.0Z"
+      datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z"
+    step: "10000d"
+    cursor_field: "{{ options.stream_cursor_field }}"
+    start_time_option:
+      field_name: "start_time"
+      inject_into: "request_parameter"
+    end_time_option:
+      field_name: "end_time"
+      inject_into: "request_parameter"
+    datetime_format: "%s"
+  messages_stream_slicer:
+    type: "DatetimeStreamSlicer"
+    start_datetime:
+      datetime: "2017-01-01T00:00:00.0Z"
+      datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z"
+    end_datetime:
+      datetime: "2022-08-01T00:00:00.0Z"
+      datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z"
+    step: "10000d"
+    cursor_field: "{{ options.stream_cursor_field }}"
+    datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z"
+
+streams:
+  - type: DeclarativeStream
+    $options:
+      name: "lists"
+      primary_key: "id"
+    schema_loader:
+      $ref: "*ref(definitions.schema_loader)"
+    retriever:
+      $ref: "*ref(definitions.retriever)"
+      record_selector:
+        $ref: "*ref(definitions.result_selector)"
+      requester:
+        $ref: "*ref(definitions.requester)"
+        path: "/v3/marketing/lists"
+      paginator:
+        $ref: "*ref(definitions.cursor_paginator)"
+  - type: DeclarativeStream
+    $options:
+      name: "campaigns"
+      primary_key: "id"
+    schema_loader:
+      $ref: "*ref(definitions.schema_loader)"
+    retriever:
+      $ref: "*ref(definitions.retriever)"
+      record_selector:
+        $ref: "*ref(definitions.result_selector)"
+      requester:
+        $ref: "*ref(definitions.requester)"
+        path: "/v3/marketing/campaigns"
+      paginator:
+        $ref: "*ref(definitions.cursor_paginator)"
+  - type: DeclarativeStream
+    $options:
+      name: "contacts"
+      primary_key: "id"
+    schema_loader:
+      $ref: "*ref(definitions.schema_loader)"
+    retriever:
+      $ref: "*ref(definitions.retriever)"
+      record_selector:
+        $ref: "*ref(definitions.result_selector)"
+      requester:
+        $ref: "*ref(definitions.requester)"
+        path: "/v3/marketing/contacts"
+      paginator:
+        type: "NoPagination"
+  - type: DeclarativeStream
+    $options:
+      name: "stats_automations"
+      primary_key: "id"
+    schema_loader:
+      $ref: "*ref(definitions.schema_loader)"
+    retriever:
+      $ref: "*ref(definitions.retriever)"
+      record_selector:
+        $ref: "*ref(definitions.results_selector)"
+      requester:
+        $ref: "*ref(definitions.requester)"
+        path: "/v3/marketing/stats/automations"
+      paginator:
+        $ref: "*ref(definitions.cursor_paginator)"
+  - type: DeclarativeStream
+    $options:
+      name: "segments"
+      primary_key: "id"
+    schema_loader:
+      $ref: "*ref(definitions.schema_loader)"
+    retriever:
+      $ref: "*ref(definitions.retriever)"
+      record_selector:
+        $ref: "*ref(definitions.results_selector)"
+      requester:
+        $ref: "*ref(definitions.requester)"
+        path: "/v3/marketing/segments"
+      paginator:
+        type: "NoPagination"
+  - type: DeclarativeStream
+    $options:
+      name: "single_sends"
+      primary_key: "id"
+    schema_loader:
+      $ref: "*ref(definitions.schema_loader)"
+    retriever:
+      $ref: "*ref(definitions.retriever)"
+      record_selector:
+        $ref: "*ref(definitions.results_selector)"
+      requester:
+        $ref: "*ref(definitions.requester)"
+        path: "/v3/marketing/stats/singlesends"
+      paginator:
+        $ref: "*ref(definitions.cursor_paginator)"
+  - type: DeclarativeStream
+    $options:
+      name: "templates"
+      primary_key: "id"
+    schema_loader:
+      $ref: "*ref(definitions.schema_loader)"
+    retriever:
+      $ref: "*ref(definitions.retriever)"
+      record_selector:
+        $ref: "*ref(definitions.result_selector)"
+      requester:
+        $ref: "*ref(definitions.requester)"
+        path: "/v3/templates"
+        request_options_provider:
+          request_parameters:
+            generations: "legacy,dynamic"
+      paginator:
+        $ref: "*ref(definitions.cursor_paginator)"
+  - type: DeclarativeStream
+    $options:
+      name: "bounces"
+      primary_key: "email"
+      stream_cursor_field: "created"
+    schema_loader:
+      $ref: "*ref(definitions.schema_loader)"
+    retriever:
+      $ref: "*ref(definitions.retriever)"
+      record_selector:
+        $ref: "*ref(definitions.selector)"
+      requester:
+        $ref: "*ref(definitions.requester)"
+        path: "/v3/suppression/bounces"
+      paginator:
+        $ref: "*ref(definitions.offset_paginator)"
+      stream_slicer:
+        $ref: "*ref(definitions.stream_slicer)"
+  - type: DeclarativeStream
+    $options:
+      name: "global_suppressions"
+      primary_key: "email"
+      stream_cursor_field: "created"
+    schema_loader:
+      $ref: "*ref(definitions.schema_loader)"
+    retriever:
+      $ref: "*ref(definitions.retriever)"
+      record_selector:
+        $ref: "*ref(definitions.selector)"
+      requester:
+        $ref: "*ref(definitions.requester)"
+        path: "/v3/suppression/unsubscribes"
+      paginator:
+        $ref: "*ref(definitions.offset_paginator)"
+      stream_slicer:
+        $ref: "*ref(definitions.stream_slicer)"
+  - type: DeclarativeStream
+    $options:
+      name: "blocks"
+      primary_key: "email"
+      stream_cursor_field: "created"
+    schema_loader:
+      $ref: "*ref(definitions.schema_loader)"
+    retriever:
+      $ref: "*ref(definitions.retriever)"
+      record_selector:
+        $ref: "*ref(definitions.selector)"
+      requester:
+        $ref: "*ref(definitions.requester)"
+        path: "/v3/suppression/blocks"
+      paginator:
+        $ref: "*ref(definitions.offset_paginator)"
+      stream_slicer:
+        $ref: "*ref(definitions.stream_slicer)"
+  - type: DeclarativeStream
+    $options:
+      name: "suppression_groups"
+      primary_key: "id"
+    schema_loader:
+      $ref: "*ref(definitions.schema_loader)"
+    retriever:
+      $ref: "*ref(definitions.retriever)"
+      record_selector:
+        $ref: "*ref(definitions.selector)"
+      requester:
+        $ref: "*ref(definitions.requester)"
+        path: "/v3/asm/groups"
+      paginator:
+        type: "NoPagination"
+  - type: DeclarativeStream
+    $options:
+      name: "suppression_group_members"
+      primary_key: "group_id"
+    schema_loader:
+      $ref: "*ref(definitions.schema_loader)"
+    retriever:
+      $ref: "*ref(definitions.retriever)"
+      record_selector:
+        $ref: "*ref(definitions.selector)"
+      requester:
+        $ref: "*ref(definitions.requester)"
+        path: "/v3/asm/suppressions"
+      paginator:
+        $ref: "*ref(definitions.offset_paginator)"
+  - type: DeclarativeStream
+    $options:
+      name: "invalid_emails"
+      primary_key: "email"
+      stream_cursor_field: "created"
+    schema_loader:
+      $ref: "*ref(definitions.schema_loader)"
+    retriever:
+      $ref: "*ref(definitions.retriever)"
+      record_selector:
+        $ref: "*ref(definitions.selector)"
+      requester:
+        $ref: "*ref(definitions.requester)"
+        path: "/v3/suppression/invalid_emails"
+      paginator:
+        $ref: "*ref(definitions.offset_paginator)"
+      stream_slicer:
+        $ref: "*ref(definitions.stream_slicer)"
+  - type: DeclarativeStream
+    $options:
+      name: "spam_reports"
+      primary_key: "email"
+      stream_cursor_field: "created"
+    schema_loader:
+      $ref: "*ref(definitions.schema_loader)"
+    retriever:
+      $ref: "*ref(definitions.retriever)"
+      record_selector:
+        $ref: "*ref(definitions.selector)"
+      requester:
+        $ref: "*ref(definitions.requester)"
+        path: "/v3/suppression/spam_reports"
+
+      paginator:
+        $ref: "*ref(definitions.offset_paginator)"
+      stream_slicer:
+        $ref: "*ref(definitions.stream_slicer)"
+  - type: DeclarativeStream
+    $options:
+      name: "messages"
+      primary_key: "msg_id"
+      stream_cursor_field: "last_event_time"
+    schema_loader:
+      $ref: "*ref(definitions.schema_loader)"
+    retriever:
+      $ref: "*ref(definitions.retriever)"
+      record_selector:
+        $ref: "*ref(definitions.selector)"
+      requester:
+        $ref: "*ref(definitions.requester)"
+        path: "/v3/messages"
+        request_options_provider:
+          request_parameters:
+            limit: 1000
+            query: 'last_event_time BETWEEN TIMESTAMP "{{stream_slice.start_time}}" AND TIMESTAMP "{{stream_slice.end_time}}"'
+      stream_slicer:
+        $ref: "*ref(definitions.messages_stream_slicer)"
+check:
+  type: CheckStream
+  stream_names: ["lists"]