✨ Source Sendgrid: add unsub groups stream (#26314)

* add unsubscribe groups stream * add doc * fix tests * fix tests * fix * fix update version * Automated Change * Delete oss_registry.json * add expected records * add eof * small fix --------- Co-authored-by: Haithem Souala <haithem.souala@woopit.fr>
airbytehq · May 23, 2023 · a89fd05 · a89fd05
1 parent 13cacdb
commit a89fd05
Show file tree

Hide file tree

Showing 10 changed files with 56 additions and 3 deletions.
diff --git a/airbyte-integrations/connectors/source-sendgrid/Dockerfile b/airbyte-integrations/connectors/source-sendgrid/Dockerfile
@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.3.1
+LABEL io.airbyte.version=0.4.0
 LABEL io.airbyte.name=airbyte/source-sendgrid
diff --git a/airbyte-integrations/connectors/source-sendgrid/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-sendgrid/integration_tests/configured_catalog.json
@@ -140,6 +140,15 @@
       "sync_mode": "incremental",
       "cursor_field": ["created"],
       "destination_sync_mode": "append"
+    },    
+    {
+      "stream": {
+        "name": "unsubscribe_groups",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
     }
   ]
 }
diff --git a/airbyte-integrations/connectors/source-sendgrid/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-sendgrid/integration_tests/expected_records.jsonl
@@ -232,3 +232,8 @@
 {"stream": "segments", "data": {"id": "bb3309ca-c002-4085-8d3e-de142dfa317e", "name": "Everyone named Jim, JIM or jim", "contacts_count": 0, "sample_updated_at": "2023-03-14T08:31:18Z", "next_sample_update": "2023-03-15T08:31:18Z", "created_at": "2021-02-03T13:19:50Z", "updated_at": "2021-02-03T13:19:50Z", "parent_list_id": "0236d6d2-75d2-42c5-962d-603e0deaf8d1", "status": {"query_validation": "VALID"}}, "emitted_at": 1678791752069}
 {"stream": "segments", "data": {"id": "414d2eed-b2ba-4260-b3b4-e66369fafa43", "name": "Everyone named Bob, BOB or bob", "contacts_count": 0, "sample_updated_at": "2023-03-14T03:31:01Z", "next_sample_update": "2023-03-15T03:31:01Z", "created_at": "2021-02-03T13:18:50Z", "updated_at": "2021-02-03T13:18:50Z", "parent_list_id": "0236d6d2-75d2-42c5-962d-603e0deaf8d1", "status": {"query_validation": "VALID"}}, "emitted_at": 1678791752070}
 {"stream": "stats_automations", "data": {"id": "85e3874b-6654-11eb-8a29-0e707ab27408", "step_id": "all", "aggregation": "total", "stats": {"bounce_drops": 0, "bounces": 6, "clicks": 0, "unique_clicks": 0, "delivered": 6, "invalid_emails": 0, "opens": 7, "unique_opens": 3, "requests": 12, "spam_report_drops": 0, "spam_reports": 0, "unsubscribes": 0}}, "emitted_at": 1678791758456}
+{"stream": "unsubscribe_groups", "data": {"name": "Test Suggestions Group 0", "id": 14760, "description": "Suggestions for testing new stream.", "is_default": false, "unsubscribes": 0}, "emitted_at": 1684520573043}
+{"stream": "unsubscribe_groups", "data": {"name": "Test Suggestions Group 1", "id": 14761, "description": "Suggestions for testing new stream.", "is_default": false, "unsubscribes": 0}, "emitted_at": 1684520573043}
+{"stream": "unsubscribe_groups", "data": {"name": "Test Suggestions Group 2", "id": 14762, "description": "Suggestions for testing new stream.", "is_default": false, "unsubscribes": 0}, "emitted_at": 1684520573044}
+{"stream": "unsubscribe_groups", "data": {"name": "Test Suggestions Group 3", "id": 14763, "description": "Suggestions for testing new stream.", "is_default": false, "unsubscribes": 0}, "emitted_at": 1684520573044}
+{"stream": "unsubscribe_groups", "data": {"name": "Test Suggestions Group 4", "id": 14764, "description": "Suggestions for testing new stream.", "is_default": false, "unsubscribes": 0}, "emitted_at": 1684520573044}
diff --git a/airbyte-integrations/connectors/source-sendgrid/metadata.yaml b/airbyte-integrations/connectors/source-sendgrid/metadata.yaml
@@ -5,7 +5,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87
-  dockerImageTag: 0.3.1
+  dockerImageTag: 0.4.0
   dockerRepository: airbyte/source-sendgrid
   githubIssueLabel: source-sendgrid
   icon: sendgrid.svg

diff --git a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/manifest.yaml b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/manifest.yaml
@@ -246,6 +246,12 @@ streams:
         request_parameters:
           limit: "1000"
           query: 'last_event_time BETWEEN TIMESTAMP "{{stream_slice.start_time}}" AND TIMESTAMP "{{stream_slice.end_time}}"'
+  - $ref: "#/definitions/base_stream"
+    $parameters:
+      name: "unsubscribe_groups"
+      primary_key: "id"
+      path: "/v3/asm/groups"
+      field_path: []
 check:
   type: CheckStream
   stream_names: ["lists"]
diff --git a/...e-integrations/connectors/source-sendgrid/source_sendgrid/schemas/unsubscribe_groups.json b/...e-integrations/connectors/source-sendgrid/source_sendgrid/schemas/unsubscribe_groups.json
@@ -0,0 +1,25 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "additionalProperties": true,
+  "type": "object",
+  "properties": {
+    "id": {
+      "type": ["null", "integer"]
+    },
+    "name": {
+      "type": ["null", "string"]
+    },
+    "description": {
+      "type": ["null", "string"]
+    },
+    "last_email_sent_at": {
+      "type": ["null", "integer"]
+    },
+    "is_default": {
+      "type": ["null", "boolean"]
+    },
+    "unsubscribes": {
+      "type": ["null", "integer"]
+    }
+  }
+}
diff --git a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/source.py b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/source.py
@@ -27,6 +27,7 @@
     SuppressionGroupMembers,
     SuppressionGroups,
     Templates,
+    UnsubscribeGroups,
 )
 
 
@@ -64,6 +65,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
             Bounces(authenticator=authenticator, start_time=start_time),
             InvalidEmails(authenticator=authenticator, start_time=start_time),
             SpamReports(authenticator=authenticator, start_time=start_time),
+            UnsubscribeGroups(authenticator=authenticator),
         ]
 
         return streams
diff --git a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/streams.py
@@ -415,3 +415,8 @@ class SpamReports(SendgridStreamOffsetPagination, SendgridStreamIncrementalMixin
 
     def path(self, **kwargs) -> str:
         return "suppression/spam_reports"
+
+
+class UnsubscribeGroups(SendgridStream):
+    def path(self, **kwargs) -> str:
+        return "asm/groups"
diff --git a/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py
@@ -65,7 +65,7 @@ def test_source_wrong_credentials():
 def test_streams():
     streams = SourceSendgrid().streams(config={"apikey": "wrong.api.key123", "start_time": FAKE_NOW_ISO_STRING})
 
-    assert len(streams) == 14
+    assert len(streams) == 15
 
 
 @patch.multiple(SendgridStreamOffsetPagination, __abstractmethods__=set())

diff --git a/docs/integrations/sources/sendgrid.md b/docs/integrations/sources/sendgrid.md
@@ -73,6 +73,7 @@ The connector is restricted by normal Sendgrid [requests limitation](https://sen
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                                                                                                                                                           |
 |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.4.0   | 2023-05-19 | [23959](https://github.com/airbytehq/airbyte/pull/23959) | Add `unsubscribe_groups`stream 
 | 0.3.1   | 2023-01-27 | [21939](https://github.com/airbytehq/airbyte/pull/21939) | Fix contacts missing records; Remove Messages stream                                                                                                                                                                                              |
 | 0.3.0   | 2023-01-25 | [21587](https://github.com/airbytehq/airbyte/pull/21587) | Make sure spec works as expected in UI - make start_time parameter an ISO string instead of an integer interpreted as timestamp (breaking, update your existing connections and set the start_time parameter to ISO 8601 date time string in UTC) |
 | 0.2.16  | 2022-11-02 | [18847](https://github.com/airbytehq/airbyte/pull/18847) | Skip the stream on `400, 401 - authorization required` with log message                                                                                                                                                                           |