Mailgun

README.md

@@ -15,6 +15,7 @@ Deploy this project to your Tinybird account and configure the webhooks for the
 - [GitLab](https://www.tinybird.co/docs/get-data-in/guides/ingest-from-gitlab)
 - [GitHub](https://www.tinybird.co/docs/get-data-in/guides/ingest-from-github)
 - [Knock](https://www.tinybird.co/docs/get-data-in/guides/ingest-from-knock)
+- [Mailgun](https://www.tinybird.co/docs/get-data-in/guides/ingest-from-mailgun)
 - [Orb](https://www.tinybird.co/docs/get-data-in/guides/ingest-from-orb)
 - [PagerDuty](https://www.tinybird.co/docs/get-data-in/guides/ingest-from-pagerduty)
 - [Resend](https://www.tinybird.co/docs/get-data-in/guides/ingest-from-resend)

tinybird/datasources/fixtures/mailgun.ndjson

@@ -0,0 +1,8 @@
+{"record":{"event-data":{"campaigns":[],"client-info":{"client-name":"Chrome","client-os":"Linux","client-type":"browser","device-type":"desktop","user-agent":"Mozilla\/5.0 (X11; Linux x86_64) AppleWebKit\/537.31 (KHTML, like Gecko) Chrome\/26.0.1410.43 Safari\/537.31"},"event":"unsubscribed","geolocation":{"city":"San Francisco","country":"US","region":"CA"},"id":"Ase7i2zsRYeDXztHGENqRA","ip":"50.56.129.169","log-level":"info","message":{"headers":{"message-id":"20130503182626.18666.16540@sandbox9deed6dd19664ce78685158dd27d1a61.mailgun.org"}},"recipient":"alice@example.com","recipient-domain":"example.com","tags":["my_tag_1","my_tag_2"],"timestamp":1521243339.873676,"user-variables":{"my-var-2":"awesome","my_var_1":"Mailgun Variable #1"}},"signature":{"signature":"fb8da22f6a25211391d3ae06a3a7db0046a3b47a0eb9951be5b81cef8347aa68","timestamp":"1734694146","token":"4b842ac79b5bd878c8832540c5c52e92d54f2b3fb4d8c87e19"}}}
+{"record":{"event-data":{"campaigns":[],"delivery-status":{"attempt-no":1,"certificate-verified":true,"code":452,"description":"","enhanced-code":"4.2.2","message":"4.2.2 The email account that you tried to reach is over quota. Please direct\n4.2.2 the recipient to\n4.2.2  https:\/\/support.example.com\/mail\/?p=422","mx-host":"smtp-in.example.com","retry-seconds":600,"session-seconds":0.1281740665435791,"tls":true,"utf8":true},"envelope":{"sender":"bob@sandbox9deed6dd19664ce78685158dd27d1a61.mailgun.org","sending-ip":"209.61.154.250","targets":"alice@example.com","transport":"smtp"},"event":"failed","flags":{"is-authenticated":true,"is-routed":false,"is-system-test":false,"is-test-mode":false},"id":"Fs7-5t81S2ispqxqDw2U4Q","log-level":"warn","message":{"attachments":[],"headers":{"from":"Bob <bob@sandbox9deed6dd19664ce78685158dd27d1a61.mailgun.org>","message-id":"20130503182626.18666.16540@sandbox9deed6dd19664ce78685158dd27d1a61.mailgun.org","subject":"Test delivered webhook","to":"Alice <alice@example.com>"},"size":111},"reason":"generic","recipient":"alice@example.com","recipient-domain":"example.com","severity":"temporary","storage":{"key":"message_key","url":"https:\/\/se.api.mailgun.net\/v3\/domains\/sandbox9deed6dd19664ce78685158dd27d1a61.mailgun.org\/messages\/message_key"},"tags":["my_tag_1","my_tag_2"],"timestamp":1521472262.908181,"user-variables":{"my-var-2":"awesome","my_var_1":"Mailgun Variable #1"}},"signature":{"signature":"a49d2f4fd019036940b57b933e36e0d7a1d66a3e7e463055e14d06871cc3ab28","timestamp":"1734694140","token":"53e6bd3145b0cee53ae978f8d395bc808c4ba5eaeccfc0e845"}}}
+{"record":{"event-data":{"api-key-id":"aff8axxx-23990xxx","envelope":{"sender":"bob@sandbox9deed6dd19664ce78685158dd27d1a61.mailgun.org","targets":"alice@example.com","transport":"smtp"},"event":"accepted","flags":{"is-authenticated":true,"is-test-mode":false},"id":"nIKIiE5URaSr-8WsuiCrBB","log-level":"info","message":{"attachments":[],"headers":{"from":"Bob <bob@sandbox9deed6dd19664ce78685158dd27d1a61.mailgun.org>","message-id":"20130503182626.18666.16540@sandbox9deed6dd19664ce78685158dd27d1a61.mailgun.org","subject":"Test accepted webhook","to":"Alice <alice@example.com>"},"size":256},"method":"HTTP","recipient":"alice@example.com","recipient-domain":"example.com","storage":{"key":"message_key","url":"https:\/\/se.api.mailgun.net\/v3\/domains\/sandbox9deed6dd19664ce78685158dd27d1a61.mailgun.org\/messages\/message_key"},"tags":["my_tag_1","my_tag_2"],"timestamp":1521472262.908181,"user-variables":{"my-var-2":"awesome","my_var_1":"Mailgun Variable #1"}},"signature":{"signature":"76c7f42d31ca435a5102656d198b83b06d426a8ee30e7c329f3001a2c127fbe7","timestamp":"1734694111","token":"2a054c00f130991c81ad6ae29610347e7641f86f758a396817"}}}
+{"record":{"event-data":{"campaigns":[],"client-info":{"client-name":"Chrome","client-os":"Linux","client-type":"browser","device-type":"desktop","user-agent":"Mozilla\/5.0 (X11; Linux x86_64) AppleWebKit\/537.31 (KHTML, like Gecko) Chrome\/26.0.1410.43 Safari\/537.31"},"event":"clicked","geolocation":{"city":"San Francisco","country":"US","region":"CA"},"id":"Ase7i2zsRYeDXztHGENqRA","ip":"50.56.129.169","log-level":"info","message":{"headers":{"message-id":"20130503182626.18666.16540@sandbox9deed6dd19664ce78685158dd27d1a61.mailgun.org"}},"recipient":"alice@example.com","recipient-domain":"example.com","tags":["my_tag_1","my_tag_2"],"timestamp":1521243339.873676,"user-variables":{"my-var-2":"awesome","my_var_1":"Mailgun Variable #1"}},"signature":{"signature":"6a7ca43d70b0900889c3a9205ac97345e61887289413e9c6f5752a0eac15323e","timestamp":"1734694116","token":"5947c8557e8cb9314a0659c65d3150c82a5c707ee9a161eb69"}}}
+{"record":{"event-data":{"campaigns":[],"envelope":{"sending-ip":"173.193.210.33"},"event":"complained","flags":{"is-test-mode":false},"id":"-Agny091SquKnsrW2NEKUA","log-level":"warn","message":{"attachments":[],"headers":{"from":"Bob <bob@sandbox9deed6dd19664ce78685158dd27d1a61.mailgun.org>","message-id":"20110215055645.25246.63817@sandbox9deed6dd19664ce78685158dd27d1a61.mailgun.org","subject":"Test complained webhook","to":"Alice <alice@example.com>"},"size":111},"recipient":"alice@example.com","tags":["my_tag_1","my_tag_2"],"timestamp":1521233123.501324,"user-variables":{"my-var-2":"awesome","my_var_1":"Mailgun Variable #1"}},"signature":{"signature":"1081d1f1d24626119267a2d713babeaaf13fdbb51748340041d133704342a5d1","timestamp":"1734694122","token":"d02c3e2d4349c623c209f7b22339e3614a81a7258274f2de7d"}}}
+{"record":{"event-data":{"campaigns":[],"delivery-status":{"attempt-no":1,"certificate-verified":true,"code":250,"description":"","message":"OK","mx-host":"smtp-in.example.com","session-seconds":0.4331989288330078,"tls":true,"utf8":true},"envelope":{"sender":"bob@sandbox9deed6dd19664ce78685158dd27d1a61.mailgun.org","sending-ip":"209.61.154.250","targets":"alice@example.com","transport":"smtp"},"event":"delivered","flags":{"is-authenticated":true,"is-routed":false,"is-system-test":false,"is-test-mode":false},"id":"CPgfbmQMTCKtHW6uIWtuVe","log-level":"info","message":{"attachments":[],"headers":{"from":"Bob <bob@sandbox9deed6dd19664ce78685158dd27d1a61.mailgun.org>","message-id":"20130503182626.18666.16540@sandbox9deed6dd19664ce78685158dd27d1a61.mailgun.org","subject":"Test delivered webhook","to":"Alice <alice@example.com>"},"size":111},"recipient":"alice@example.com","recipient-domain":"example.com","storage":{"key":"message_key","url":"https:\/\/se.api.mailgun.net\/v3\/domains\/sandbox9deed6dd19664ce78685158dd27d1a61.mailgun.org\/messages\/message_key"},"tags":["my_tag_1","my_tag_2"],"timestamp":1521472262.908181,"user-variables":{"my-var-2":"awesome","my_var_1":"Mailgun Variable #1"}},"signature":{"signature":"d2379482604b9ad72f1ebcf6129482634a14b474c062c31a894e88ae7a6775ad","timestamp":"1734694127","token":"fd3f0a061eab6b43c49beacb3e7512d15220f90c2f14a30a6b"}}}
+{"record":{"event-data":{"campaigns":[],"client-info":{"client-name":"Chrome","client-os":"Linux","client-type":"browser","device-type":"desktop","user-agent":"Mozilla\/5.0 (X11; Linux x86_64) AppleWebKit\/537.31 (KHTML, like Gecko) Chrome\/26.0.1410.43 Safari\/537.31"},"event":"opened","geolocation":{"city":"San Francisco","country":"US","region":"CA"},"id":"Ase7i2zsRYeDXztHGENqRA","ip":"50.56.129.169","log-level":"info","message":{"headers":{"message-id":"20130503182626.18666.16540@sandbox9deed6dd19664ce78685158dd27d1a61.mailgun.org"}},"recipient":"alice@example.com","recipient-domain":"example.com","tags":["my_tag_1","my_tag_2"],"timestamp":1521243339.873676,"user-variables":{"my-var-2":"awesome","my_var_1":"Mailgun Variable #1"}},"signature":{"signature":"e1eb8226091abebee1463d56f2d7b0793348edd18a805dc1d5e4fd5175d15f46","timestamp":"1734694133","token":"8b9070c44c5abb542fdc047d79fdb8b147b43f91dd4a0bc6eb"}}}
+{"record":{"event-data":{"campaigns":[],"delivery-status":{"attempt-no":1,"code":605,"description":"Not delivering to previously bounced address","enhanced-code":"","message":"","session-seconds":0},"envelope":{"sender":"bob@sandbox9deed6dd19664ce78685158dd27d1a61.mailgun.org","targets":"alice@example.com","transport":"smtp"},"event":"failed","flags":{"is-authenticated":true,"is-routed":false,"is-system-test":false,"is-test-mode":false},"id":"G9Bn5sl1TC6nu79C8C0bwg","log-level":"error","message":{"attachments":[],"headers":{"from":"Bob <bob@sandbox9deed6dd19664ce78685158dd27d1a61.mailgun.org>","message-id":"20130503192659.13651.20287@sandbox9deed6dd19664ce78685158dd27d1a61.mailgun.org","subject":"Test permanent_fail webhook","to":"Alice <alice@example.com>"},"size":111},"reason":"suppress-bounce","recipient":"alice@example.com","recipient-domain":"example.com","severity":"permanent","storage":{"key":"message_key","url":"https:\/\/se.api.mailgun.net\/v3\/domains\/sandbox9deed6dd19664ce78685158dd27d1a61.mailgun.org\/messages\/message_key"},"tags":["my_tag_1","my_tag_2"],"timestamp":1521233195.375624,"user-variables":{"my-var-2":"awesome","my_var_1":"Mailgun Variable #1"}},"signature":{"signature":"718f994bd98e108902add9007d8cd5b5109b4da1876dddeba147581939b129dc","timestamp":"1734694138","token":"a363ef9931a72a6c4a0f1024f50b0cd10b0ec60d412adf6383"}}}

tinybird/datasources/mailgun.datasource

@@ -0,0 +1,25 @@
+TOKEN "append" APPEND
+
+# Refer to https://mailgun-docs.redoc.ly/docs/mailgun/user-manual/events/#event-structure
+
+TAGS "mailgun"
+
+SCHEMA >
+    `date` DateTime `json:$.record.event-data.timestamp`,
+    `event` LowCardinality(String) `json:$.record.event-data.event`,
+    `message_id` String `json:$.record.event-data.message.headers.message-id` DEFAULT '',
+    `recipient` String `json:$.record.event-data.recipient` DEFAULT '',
+    `recipient_domain` String `json:$.record.event-data.recipient-domain` DEFAULT '',
+    `tags` Array(String) `json:$.record.event-data.tags[:]` DEFAULT [],
+    `log_level` LowCardinality(String) `json:$.record.event-data.log-level` DEFAULT '',
+    `city` String `json:$.record.event-data.geolocation.city` DEFAULT '',
+    `country` String `json:$.record.event-data.geolocation.country` DEFAULT '',
+    `region` String `json:$.record.event-data.geolocation.region` DEFAULT '',
+    `client_info` JSON `json:$.record.event-data.client-info` DEFAULT '{}',
+    `delivery_status` JSON `json:$.record.event-data.delivery-status` DEFAULT '{}',
+    `envelope` JSON `json:$.record.event-data.envelope` DEFAULT '{}',
+    `user_variables` JSON `json:$.record.event-data.user-variables` DEFAULT '{}'
+
+ENGINE "MergeTree"
+ENGINE_PARTITION_KEY "toYYYYMM(date)"
+ENGINE_SORTING_KEY "event, date"