🚨🚨 Source Sendgrid: migrate to low code (#35776)

Co-authored-by: askarpets <anton.karpets@globallogic.com>
Co-authored-by: Ella Rohm-Ensing <erohmensing@gmail.com>

airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py

@@ -299,6 +299,8 @@ async def discovered_catalog_fixture(
 
     output = await docker_runner.call_discover(config=connector_config)
     catalogs = [message.catalog for message in output if message.type == Type.CATALOG]
+    if len(catalogs) == 0:
+        raise ValueError("No catalog message was emitted")
     return {stream.name: stream for stream in catalogs[-1].streams}
 
 
@@ -322,6 +324,8 @@ async def previous_discovered_catalog_fixture(
         )
         return None
     catalogs = [message.catalog for message in output if message.type == Type.CATALOG]
+    if len(catalogs) == 0:
+        raise ValueError("No catalog message was emitted")
     return {stream.name: stream for stream in catalogs[-1].streams}
 
 

airbyte-integrations/connectors/source-sendgrid/.coveragerc

@@ -1,3 +1,4 @@
 [run]
 omit = 
     source_sendgrid/run.py
+    source_sendgrid/__init__.py

airbyte-integrations/connectors/source-sendgrid/README.md

@@ -30,7 +30,7 @@ See `sample_files/sample_config.json` for a sample config file.
 poetry run source-sendgrid spec
 poetry run source-sendgrid check --config secrets/config.json
 poetry run source-sendgrid discover --config secrets/config.json
-poetry run source-sendgrid read --config secrets/config.json --catalog sample_files/configured_catalog.json
+poetry run source-sendgrid read --config secrets/config.json --catalog integration_tests/configured_catalog.json
 ```
 
 ### Running unit tests

airbyte-integrations/connectors/source-sendgrid/acceptance-test-config.yml

@@ -4,26 +4,35 @@ acceptance_tests:
   spec:
     tests:
       - spec_path: "source_sendgrid/spec.json"
+        config_path: "secrets/lowcode_config.json"
         backward_compatibility_tests_config:
-          disable_for_version: "0.3.0"
+          disable_for_version: "0.5.0"
   connection:
     tests:
-      - config_path: "secrets/config.json"
+      - config_path: "secrets/lowcode_config.json"
         status: "succeed"
       - config_path: "secrets/old_config.json"
         status: "succeed"
+      - config_path: "secrets/python_config.json"
+        status: "succeed"
       - config_path: "integration_tests/invalid_time.json"
         status: "failed"
       - config_path: "integration_tests/invalid_api_key.json"
         status: "failed"
   discovery:
     tests:
+      - config_path: "secrets/lowcode_config.json"
+        backward_compatibility_tests_config:
+          disable_for_version: "1.0.0"
       - config_path: "secrets/config.json"
         backward_compatibility_tests_config:
           disable_for_version: "0.3.0"
+      - config_path: "secrets/python_config.json"
+        backward_compatibility_tests_config:
+          disable_for_version: "0.3.0"
   basic_read:
     tests:
-      - config_path: "secrets/config.json"
+      - config_path: "secrets/lowcode_config.json"
         expect_records:
           path: "integration_tests/expected_records.jsonl"
           exact_order: no
@@ -37,11 +46,11 @@ acceptance_tests:
         fail_on_extra_columns: false
   incremental:
     tests:
-      - config_path: "secrets/config.json"
+      - config_path: "secrets/lowcode_config.json"
         configured_catalog_path: "integration_tests/no_spam_reports_configured_catalog.json"
         future_state:
           future_state_path: "integration_tests/abnormal_state.json"
   full_refresh:
     tests:
-      - config_path: "secrets/config.json"
+      - config_path: "secrets/lowcode_config.json"
         configured_catalog_path: "integration_tests/configured_catalog.json"

airbyte-integrations/connectors/source-sendgrid/integration_tests/abnormal_state.json

@@ -42,5 +42,16 @@
         "name": "invalid_emails"
       }
     }
+  },
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {
+        "created_at": "7270247822"
+      },
+      "stream_descriptor": {
+        "name": "suppression_group_members"
+      }
+    }
   }
 ]

airbyte-integrations/connectors/source-sendgrid/integration_tests/configured_catalog.json

@@ -47,7 +47,16 @@
     },
     {
       "stream": {
-        "name": "single_sends",
+        "name": "singlesends",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "singlesend_stats",
         "json_schema": {},
         "supported_sync_modes": ["full_refresh"]
       },
@@ -140,15 +149,6 @@
       "sync_mode": "incremental",
       "cursor_field": ["created"],
       "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "unsubscribe_groups",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh"]
-      },
-      "sync_mode": "full_refresh",
-      "destination_sync_mode": "overwrite"
     }
   ]
 }

airbyte-integrations/connectors/source-sendgrid/integration_tests/expected_records.jsonl

@@ -104,8 +104,8 @@
 {"stream":"contacts","data":{"email":"fake_email_15@lmail.c","first_name":"Fake contact","last_name":"Lastname","address_line_1":null,"address_line_2":null,"city":null,"state_province_region":null,"postal_code":"22341","country":null,"alternate_emails":null,"phone_number":null,"whatsapp":null,"line":null,"facebook":null,"unique_name":null,"created_at":"2021-02-01T12:35:27Z","updated_at":"2021-02-01T12:35:56Z","contact_id":"50b36a31-daf8-45c4-bc48-13e150f6746e"},"emitted_at":1674825613540}
 {"stream":"contacts","data":{"email":"fake_email_3@lmail.c","first_name":"Fake contact","last_name":"Lastname","address_line_1":null,"address_line_2":null,"city":null,"state_province_region":null,"postal_code":"22341","country":null,"alternate_emails":null,"phone_number":null,"whatsapp":null,"line":null,"facebook":null,"unique_name":null,"created_at":"2021-02-01T12:35:14Z","updated_at":"2021-02-01T12:35:43Z","contact_id":"d1211b88-e116-4a0b-a823-0361bf059a06"},"emitted_at":1674825613540}
 {"stream":"contacts","data":{"email":"fake_email_18@lmail.c","first_name":"Fake contact","last_name":"Lastname","address_line_1":null,"address_line_2":null,"city":null,"state_province_region":null,"postal_code":"22341","country":null,"alternate_emails":null,"phone_number":null,"whatsapp":null,"line":null,"facebook":null,"unique_name":null,"created_at":"2021-02-01T12:35:30Z","updated_at":"2021-02-01T12:36:00Z","contact_id":"19163421-bb29-495d-950f-edede6218081"},"emitted_at":1674825613540}
-{"stream": "single_sends", "data": {"ab_phase": "all", "ab_variation": "all", "aggregation": "total", "stats": {"bounce_drops": 0, "bounces": 0, "clicks": 0, "unique_clicks": 0, "delivered": 1, "invalid_emails": 0, "opens": 3, "unique_opens": 1, "requests": 1, "spam_report_drops": 0, "spam_reports": 0, "unsubscribes": 1}, "id": "3c5a9fa6-1084-11ec-ac32-4228d699bad5"}, "emitted_at": 1631093373000}
-{"stream": "single_sends", "data": {"ab_phase": "all", "ab_variation": "all", "aggregation": "total", "stats": {"bounce_drops": 0, "bounces": 0, "clicks": 1, "unique_clicks": 1, "delivered": 1, "invalid_emails": 0, "opens": 1, "unique_opens": 1, "requests": 1, "spam_report_drops": 0, "spam_reports": 0, "unsubscribes": 0}, "id": "c9f286fb-1083-11ec-ae03-ca0fc7f28419"}, "emitted_at": 1631093373000}
+{"stream": "singlesend_stats", "data": {"ab_phase": "all", "ab_variation": "all", "aggregation": "total", "stats": {"bounce_drops": 0, "bounces": 0, "clicks": 1, "unique_clicks": 1, "delivered": 1, "invalid_emails": 0, "opens": 1, "unique_opens": 1, "requests": 1, "spam_report_drops": 0, "spam_reports": 0, "unsubscribes": 0}, "id": "c9f286fb-1083-11ec-ae03-ca0fc7f28419"}, "emitted_at": 1710090955628}
+{"stream": "singlesend_stats", "data": {"ab_phase": "all", "ab_variation": "all", "aggregation": "total", "stats": {"bounce_drops": 0, "bounces": 0, "clicks": 0, "unique_clicks": 0, "delivered": 1, "invalid_emails": 0, "opens": 3, "unique_opens": 1, "requests": 1, "spam_report_drops": 0, "spam_reports": 0, "unsubscribes": 1}, "id": "3c5a9fa6-1084-11ec-ac32-4228d699bad5"}, "emitted_at": 1710090955627}
 {"stream": "templates", "data": {"id": "d-9e5be59949e043e69f3df003e715b99b", "name": "test_dynamic_template", "generation": "dynamic", "updated_at": "2021-02-03 14:56:56", "versions": []}, "emitted_at": 1631093374000}
 {"stream": "templates", "data": {"id": "93702a5d-4cb2-4616-8d30-c2766e2b8e18", "name": "Template number 20", "generation": "legacy", "updated_at": "2021-02-03 13:31:29", "versions": []}, "emitted_at": 1631093374000}
 {"stream": "templates", "data": {"id": "0fd7bcc7-7459-49e9-b9f3-4ace4f9a9ae2", "name": "Template number 19", "generation": "legacy", "updated_at": "2021-02-03 13:31:28", "versions": []}, "emitted_at": 1631093374000}
@@ -212,12 +212,12 @@
 {"stream": "campaigns", "data": {"created_at": "2021-02-01T12:12:25Z", "id": "be9d147f-6486-11eb-8b51-8aa6caa37fdd", "name": "Third Campaign", "status": "draft", "updated_at": "2021-02-01T12:12:25Z", "is_abtest": false, "channels": ["email"]}, "emitted_at": 1678791750592}
 {"stream": "campaigns", "data": {"created_at": "2021-02-01T12:12:18Z", "id": "ba43f256-6486-11eb-bb4f-823d082c01b8", "name": "Second Campaign", "status": "draft", "updated_at": "2021-02-01T12:12:18Z", "is_abtest": false, "channels": ["email"]}, "emitted_at": 1678791750593}
 {"stream": "campaigns", "data": {"created_at": "2021-02-01T12:10:59Z", "id": "8b17a7b7-6486-11eb-bd77-2a301ccc59da", "name": "First Campaign", "status": "draft", "updated_at": "2021-02-01T12:10:59Z", "is_abtest": false, "channels": ["email"]}, "emitted_at": 1678791750593}
-{"stream": "segments", "data": {"id": "37d93ee7-268e-4f6f-a103-384af7324b35", "name": "Everyone named Pol, POL or pol", "contacts_count": 0, "sample_updated_at": "2023-03-13T17:30:26Z", "next_sample_update": "2023-03-14T17:30:26Z", "created_at": "2021-02-03T13:23:13Z", "updated_at": "2021-02-03T13:23:13Z", "parent_list_id": null, "status": {"query_validation": "VALID"}}, "emitted_at": 1678791752069}
-{"stream": "segments", "data": {"id": "bb3309ca-c002-4085-8d3e-de142dfa317e", "name": "Everyone named Jim, JIM or jim", "contacts_count": 0, "sample_updated_at": "2023-03-14T08:31:18Z", "next_sample_update": "2023-03-15T08:31:18Z", "created_at": "2021-02-03T13:19:50Z", "updated_at": "2021-02-03T13:19:50Z", "parent_list_id": "0236d6d2-75d2-42c5-962d-603e0deaf8d1", "status": {"query_validation": "VALID"}}, "emitted_at": 1678791752069}
-{"stream": "segments", "data": {"id": "414d2eed-b2ba-4260-b3b4-e66369fafa43", "name": "Everyone named Bob, BOB or bob", "contacts_count": 0, "sample_updated_at": "2023-03-14T03:31:01Z", "next_sample_update": "2023-03-15T03:31:01Z", "created_at": "2021-02-03T13:18:50Z", "updated_at": "2021-02-03T13:18:50Z", "parent_list_id": "0236d6d2-75d2-42c5-962d-603e0deaf8d1", "status": {"query_validation": "VALID"}}, "emitted_at": 1678791752070}
+{"stream": "segments", "data": {"id": "37d93ee7-268e-4f6f-a103-384af7324b35", "name": "Everyone named Pol, POL or pol", "query_version": "1", "contacts_count": 0, "sample_updated_at": "2024-03-09T17:30:18Z", "next_sample_update": "2024-03-10T17:30:18Z", "created_at": "2021-02-03T13:23:13Z", "updated_at": "2021-02-03T13:23:13Z", "parent_list_ids": null, "status": {"query_validation": "VALID"}}, "emitted_at": 1710091076235}
+{"stream": "segments", "data": {"id": "bb3309ca-c002-4085-8d3e-de142dfa317e", "name": "Everyone named Jim, JIM or jim", "query_version": "1", "contacts_count": 0, "sample_updated_at": "2024-03-10T08:30:38Z", "next_sample_update": "2024-03-11T08:30:38Z", "created_at": "2021-02-03T13:19:50Z", "updated_at": "2021-02-03T13:19:50Z", "parent_list_ids": ["0236d6d2-75d2-42c5-962d-603e0deaf8d1"], "status": {"query_validation": "VALID"}}, "emitted_at": 1710091076236}
+{"stream": "segments", "data": {"id": "414d2eed-b2ba-4260-b3b4-e66369fafa43", "name": "Everyone named Bob, BOB or bob", "query_version": "1", "contacts_count": 0, "sample_updated_at": "2024-03-10T03:30:41Z", "next_sample_update": "2024-03-11T03:30:41Z", "created_at": "2021-02-03T13:18:50Z", "updated_at": "2021-02-03T13:18:50Z", "parent_list_ids": ["0236d6d2-75d2-42c5-962d-603e0deaf8d1"], "status": {"query_validation": "VALID"}}, "emitted_at": 1710091076236}
 {"stream": "stats_automations", "data": {"id": "85e3874b-6654-11eb-8a29-0e707ab27408", "step_id": "all", "aggregation": "total", "stats": {"bounce_drops": 0, "bounces": 6, "clicks": 0, "unique_clicks": 0, "delivered": 6, "invalid_emails": 0, "opens": 7, "unique_opens": 3, "requests": 12, "spam_report_drops": 0, "spam_reports": 0, "unsubscribes": 0}}, "emitted_at": 1678791758456}
-{"stream": "unsubscribe_groups", "data": {"name": "Test Suggestions Group 0", "id": 14760, "description": "Suggestions for testing new stream.", "is_default": false, "unsubscribes": 0}, "emitted_at": 1684520573043}
-{"stream": "unsubscribe_groups", "data": {"name": "Test Suggestions Group 1", "id": 14761, "description": "Suggestions for testing new stream.", "is_default": false, "unsubscribes": 0}, "emitted_at": 1684520573043}
-{"stream": "unsubscribe_groups", "data": {"name": "Test Suggestions Group 2", "id": 14762, "description": "Suggestions for testing new stream.", "is_default": false, "unsubscribes": 0}, "emitted_at": 1684520573044}
-{"stream": "unsubscribe_groups", "data": {"name": "Test Suggestions Group 3", "id": 14763, "description": "Suggestions for testing new stream.", "is_default": false, "unsubscribes": 0}, "emitted_at": 1684520573044}
-{"stream": "unsubscribe_groups", "data": {"name": "Test Suggestions Group 4", "id": 14764, "description": "Suggestions for testing new stream.", "is_default": false, "unsubscribes": 0}, "emitted_at": 1684520573044}
+{"stream": "singlesends", "data": {"id": "d497b877-6486-11eb-be53-b2a243c7228c", "name": "Campaign 18", "status": "draft", "categories": [], "send_at": null, "created_at": "2021-02-01T12:13:02Z", "updated_at": "2021-02-01T12:13:02Z", "is_abtest": false, "abtest": null}, "emitted_at": 1710090864093}
+{"stream": "singlesends", "data": {"id": "c9f286fb-1083-11ec-ae03-ca0fc7f28419", "name": "Copy of Untitled Single Send", "status": "triggered", "categories": ["Promotional", "Travel & Hospitality"], "send_at": "2021-09-08T09:08:32Z", "created_at": "2021-09-08T09:04:36Z", "updated_at": "2021-09-08T09:09:08Z", "is_abtest": false, "abtest": null}, "emitted_at": 1710090864093}
+{"stream": "singlesends", "data": {"id": "4e5be6a3-1082-11ec-8512-9afd40c324e6", "name": "Untitled Single Send", "status": "triggered", "categories": ["Promotional", "Travel & Hospitality"], "send_at": "2021-09-08T08:57:05Z", "created_at": "2021-09-08T08:53:59Z", "updated_at": "2021-09-08T08:57:08Z", "is_abtest": false, "abtest": null}, "emitted_at": 1710090864093}
+{"stream": "singlesends", "data": {"id": "06ee105f-1082-11ec-8245-86a627812e3d", "name": "Untitled Single Send", "status": "triggered", "categories": [], "send_at": "2021-09-08T08:54:40Z", "created_at": "2021-09-08T08:51:59Z", "updated_at": "2021-09-08T08:55:08Z", "is_abtest": false, "abtest": null}, "emitted_at": 1710090864093}
+{"stream": "singlesends", "data": {"id": "3c5a9fa6-1084-11ec-ac32-4228d699bad5", "name": "Untitled Single Send", "status": "triggered", "categories": ["Travel & Hospitality", "Loyalty"], "send_at": "2021-09-08T09:10:22Z", "created_at": "2021-09-08T09:07:48Z", "updated_at": "2021-09-08T09:11:08Z", "is_abtest": false, "abtest": null}, "emitted_at": 1710090864093}

airbyte-integrations/connectors/source-sendgrid/integration_tests/invalid_api_key.json

@@ -1,3 +1,3 @@
 {
-  "apikey": "wrong.api.key123"
+  "api_key": "wrong.api.key123"
 }

airbyte-integrations/connectors/source-sendgrid/integration_tests/invalid_time.json

@@ -1,4 +1,4 @@
 {
-  "apikey": "apikey",
-  "start_time": "some erroneous input"
+  "api_key": "apikey",
+  "start_date": "some erroneous input"
 }

airbyte-integrations/connectors/source-sendgrid/integration_tests/no_spam_reports_configured_catalog.json

@@ -47,7 +47,16 @@
     },
     {
       "stream": {
-        "name": "single_sends",
+        "name": "singlesends",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "singlesend_stats",
         "json_schema": {},
         "supported_sync_modes": ["full_refresh"]
       },

airbyte-integrations/connectors/source-sendgrid/metadata.yaml

@@ -10,7 +10,18 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87
-  dockerImageTag: 0.5.0
+  dockerImageTag: 1.0.0
+  releases:
+    breakingChanges:
+      1.0.0:
+        message:
+          This release makes several changes that upgrade the Sendgrid connector.
+          The configuration options have been renamed to `api_key` and `start_date`. `start_date` is now required.
+          The `unsubscribe_groups` stream has been removed. It was the same as `suppression_groups`. You can use that and get the same data.
+          The `single_sends` stream has been renamed `singlesend_stats`. This is closer to the data and API.
+          The `segments` stream has been upgraded to use the Sendgrid 2.0 API because the older one has been deprecated. The schema has changed as a result.
+          To ensure a smooth upgrade, please refresh your schemas and reset your data before resuming syncs.
+        upgradeDeadline: "2024-04-29"
   dockerRepository: airbyte/source-sendgrid
   documentationUrl: https://docs.airbyte.com/integrations/sources/sendgrid
   githubIssueLabel: source-sendgrid