🐛 Source Twilio: fix failed SAT and exposed auth_token (#20430)

airbyte-integrations/connectors/source-twilio/README.md

@@ -99,7 +99,8 @@ Customize `acceptance-test-config.yml` file to configure tests. See [Source Acce
 If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
 To run your integration tests with acceptance tests, from the connector root, run
 ```
-python -m pytest integration_tests -p integration_tests.acceptance
+docker build . --no-cache -t airbyte/source-twilio:dev \
+&& python -m pytest integration_tests -p integration_tests.acceptance
 ```
 To run your integration tests with docker
 

airbyte-integrations/connectors/source-twilio/acceptance-test-config.yml

@@ -19,12 +19,20 @@ acceptance_tests:
         expect_records:
           path: "integration_tests/expected_records.txt"
         empty_streams:
+            # TODO: SAT should be able to exclude expected records validation on the stream > field level
+            # to avoid exposures such as `auth_token` 
+            # reference: https://github.com/airbytehq/airbyte/pull/20430
+          - name: "accounts"
+            bypass_reason: "expected records could not be provided without field bypassing, skipping for now. Read #TODO above."
+            # 
           - name: "alerts"
             bypass_reason: "alerts are transient - they emerge and disappear from time to time"
           - name: "dependent_phone_numbers"
             bypass_reason: "stream not filled yet"
           - name: "conference_participants"
             bypass_reason: "stream not filled yet"
+          - name: "keys"
+            bypass_reason: "stream not filled yet"
           - name: "available_phone_numbers_local"
             bypass_reason: "very volatile data"
           - name: "available_phone_numbers_mobile"
@@ -40,14 +48,17 @@ acceptance_tests:
         configured_catalog_path: "integration_tests/incremental_catalog.json"
         future_state:
           future_state_path: "integration_tests/abnormal_state.json"
+        timeout_seconds: 3600
       - config_path: "secrets/config_with_lookback.json"
         configured_catalog_path: "integration_tests/incremental_catalog.json"
         future_state:
           future_state_path: "integration_tests/abnormal_state.json"
         threshold_days: 30
+        timeout_seconds: 3600
   full_refresh:
     tests:
       - config_path: "secrets/config.json"
         # `constant_records_catalog.json` does not contain the available phone numbers streams,
         # as they may change on each request
         configured_catalog_path: "integration_tests/constant_records_catalog.json"
+        timeout_seconds: 3600

airbyte-integrations/connectors/source-twilio/integration_tests/constant_records_catalog.json

@@ -1,14 +1,5 @@
 {
   "streams": [
-    {
-      "stream": {
-        "name": "accounts",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh"]
-      },
-      "sync_mode": "full_refresh",
-      "destination_sync_mode": "overwrite"
-    },
     {
       "stream": {
         "name": "addresses",

airbyte-integrations/connectors/source-twilio/integration_tests/expected_records.txt

@@ -140,7 +140,6 @@
 {"stream": "available_phone_number_countries", "data": {"country_code": "AU", "country": "Australia", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AU.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AU/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AU/TollFree.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AU/Mobile.json"}}, "emitted_at": 1664560271163}
 {"stream": "available_phone_number_countries", "data": {"country_code": "AR", "country": "Argentina", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AR.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AR/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AR/TollFree.json"}}, "emitted_at": 1664560271163}
 {"stream": "incoming_phone_numbers", "data": {"sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "friendly_name": "2FA Number - PLEASE DO NOT TOUCH. Use another number for anythin", "phone_number": "+12056561170", "voice_url": "https://handler.twilio.com/twiml/EH7af811843f38093d724a5c2e80b3eabe", "voice_method": "POST", "voice_fallback_url": "", "voice_fallback_method": "POST", "voice_caller_id_lookup": false, "date_created": "2020-12-11T04:28:40Z", "date_updated": "2022-09-23T14:47:41Z", "sms_url": "https://webhooks.twilio.com/v1/Accounts/ACdade166c12e160e9ed0a6088226718fb/Flows/FWbd726b7110b21294a9f27a47f4ab0080", "sms_method": "POST", "sms_fallback_url": "", "sms_fallback_method": "POST", "address_requirements": "none", "beta": false, "capabilities": {"voice": true, "sms": true, "mms": true}, "status_callback": "", "status_callback_method": "POST", "api_version": "2010-04-01", "voice_application_sid": "", "sms_application_sid": "", "origin": "twilio", "trunk_sid": null, "emergency_status": "Active", "emergency_address_sid": null, "emergency_address_status": "unregistered", "address_sid": null, "identity_sid": null, "bundle_sid": null, "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/IncomingPhoneNumbers/PNe40bd7f3ac343b32fd51275d2d5b3dcc.json", "status": "in-use"}, "emitted_at": 1655893245291}
-{"stream": "keys", "data": {"date_updated": "2021-02-01T07:30:21Z", "date_created": "2021-02-01T07:30:21Z", "friendly_name": "Studio API Key", "sid": "SK60085e9cfc3d94aa1b987b25c78067a9"}, "emitted_at": 1655893247168}
 {"stream": "calls", "data": {"date_updated": "2022-06-17T22:28:34Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 61, "from": "+15312726629", "to": "+12056561170", "annotation": null, "answered_by": null, "sid": "CAe71d3c7533543b5c81b1be3fc5affa2b", "queue_time": 0, "price": -0.017, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2022-06-17T22:27:33Z", "date_created": "2022-06-17T22:27:32Z", "from_formatted": "(531) 272-6629", "group_sid": null, "trunk_sid": "", "forwarded_from": "+12056561170", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2022-06-17T22:28:34Z", "to_formatted": "(205) 656-1170", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b/Feedback.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b/Payments.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json", "user_defined_messages": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b/UserDefinedMessages.json", "user_defined_message_subscriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAe71d3c7533543b5c81b1be3fc5affa2b/UserDefinedMessageSubscriptions.json"}}, "emitted_at": 1655893249727}
 {"stream": "calls", "data": {"date_updated": "2022-06-17T13:36:17Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 96, "from": "+17372040136", "to": "+12056561170", "annotation": null, "answered_by": null, "sid": "CA0a47223735162e1a7df2738327bda2ab", "queue_time": 0, "price": -0.017, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2022-06-17T13:34:41Z", "date_created": "2022-06-17T13:34:41Z", "from_formatted": "(737) 204-0136", "group_sid": null, "trunk_sid": "", "forwarded_from": "+12056561170", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0a47223735162e1a7df2738327bda2ab.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2022-06-17T13:36:17Z", "to_formatted": "(205) 656-1170", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0a47223735162e1a7df2738327bda2ab/Feedback.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0a47223735162e1a7df2738327bda2ab/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0a47223735162e1a7df2738327bda2ab/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0a47223735162e1a7df2738327bda2ab/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0a47223735162e1a7df2738327bda2ab/Payments.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0a47223735162e1a7df2738327bda2ab/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0a47223735162e1a7df2738327bda2ab/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json", "user_defined_messages": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0a47223735162e1a7df2738327bda2ab/UserDefinedMessages.json", "user_defined_message_subscriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0a47223735162e1a7df2738327bda2ab/UserDefinedMessageSubscriptions.json"}}, "emitted_at": 1655893249739}
 {"stream": "calls", "data": {"date_updated": "2022-06-16T20:02:43Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 124, "from": "+17372040136", "to": "+12056561170", "annotation": null, "answered_by": null, "sid": "CAace5c8813c499253bbbff29ad0da0ccb", "queue_time": 0, "price": -0.0255, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2022-06-16T20:00:39Z", "date_created": "2022-06-16T20:00:39Z", "from_formatted": "(737) 204-0136", "group_sid": null, "trunk_sid": "", "forwarded_from": "+12056561170", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAace5c8813c499253bbbff29ad0da0ccb.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2022-06-16T20:02:43Z", "to_formatted": "(205) 656-1170", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAace5c8813c499253bbbff29ad0da0ccb/Feedback.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAace5c8813c499253bbbff29ad0da0ccb/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAace5c8813c499253bbbff29ad0da0ccb/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAace5c8813c499253bbbff29ad0da0ccb/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAace5c8813c499253bbbff29ad0da0ccb/Payments.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAace5c8813c499253bbbff29ad0da0ccb/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAace5c8813c499253bbbff29ad0da0ccb/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json", "user_defined_messages": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAace5c8813c499253bbbff29ad0da0ccb/UserDefinedMessages.json", "user_defined_message_subscriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAace5c8813c499253bbbff29ad0da0ccb/UserDefinedMessageSubscriptions.json"}}, "emitted_at": 1655893249745}
@@ -524,5 +523,3 @@
 {"stream": "usage_triggers", "data": {"sid": "UT33bd2bf238d94863a609133da897d676", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "date_created": "2020-11-25T10:02:19Z", "date_updated": "2020-11-25T10:02:19Z", "date_fired": null, "friendly_name": null, "usage_category": "sms", "trigger_by": "usage", "recurring": "", "trigger_value": 1000.0, "current_value": 130.0, "callback_url": "http://www.example.com/", "callback_method": "POST", "usage_record_uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Usage/Records.json?Category=sms", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Usage/Triggers/UT33bd2bf238d94863a609133da897d676.json", "api_version": "2010-04-01"}, "emitted_at": 1655893322691}
 {"stream": "usage_triggers", "data": {"sid": "UT3c3c157dcaf347829d5a0f75e97b572e", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "date_created": "2020-11-25T10:02:34Z", "date_updated": "2020-11-25T10:02:34Z", "date_fired": null, "friendly_name": null, "usage_category": "sms", "trigger_by": "usage", "recurring": "", "trigger_value": 999.0, "current_value": 130.0, "callback_url": "http://www.example.com/", "callback_method": "POST", "usage_record_uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Usage/Records.json?Category=sms", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Usage/Triggers/UT3c3c157dcaf347829d5a0f75e97b572e.json", "api_version": "2010-04-01"}, "emitted_at": 1655893322698}
 {"stream": "usage_triggers", "data": {"sid": "UT7170996eff504647ac9f215222ee296f", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "date_created": "2020-11-25T10:02:41Z", "date_updated": "2020-11-25T10:02:41Z", "date_fired": null, "friendly_name": null, "usage_category": "sms", "trigger_by": "usage", "recurring": "", "trigger_value": 943.0, "current_value": 130.0, "callback_url": "http://www.example.com/", "callback_method": "POST", "usage_record_uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Usage/Records.json?Category=sms", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Usage/Triggers/UT7170996eff504647ac9f215222ee296f.json", "api_version": "2010-04-01"}, "emitted_at": 1655893322701}
-{"stream": "accounts", "data": {"status": "active", "date_updated": "2022-07-06T23:12:05Z", "auth_token": "8fe66dc0a454921da51492e4563121d6", "friendly_name": "My first Twilio project", "owner_account_sid": "ACdade166c12e160e9ed0a6088226718fb", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb.json", "sid": "ACdade166c12e160e9ed0a6088226718fb", "date_created": "2020-11-17T04:08:53Z", "type": "Full", "subresource_uris": {"addresses": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Addresses.json", "conferences": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Conferences.json", "signing_keys": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/SigningKeys.json", "transcriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Transcriptions.json", "connect_apps": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/ConnectApps.json", "sip": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/SIP.json", "authorized_connect_apps": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AuthorizedConnectApps.json", "usage": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Usage.json", "keys": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Keys.json", "applications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Applications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Recordings.json", "short_codes": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/SMS/ShortCodes.json", "calls": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Notifications.json", "incoming_phone_numbers": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/IncomingPhoneNumbers.json", "queues": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Queues.json", "messages": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages.json", "outgoing_caller_ids": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/OutgoingCallerIds.json", "available_phone_numbers": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers.json", "balance": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Balance.json"}}, "emitted_at": 1669128692054}
-{"stream": "accounts", "data": {"status": "active", "date_updated": "2021-02-05T20:18:45Z", "auth_token": "0b7ffb8fa15e8918dee42f5d53c6da7e", "friendly_name": "SubAccount Created at 2020-11-25 01:36 AM", "owner_account_sid": "ACdade166c12e160e9ed0a6088226718fb", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee.json", "sid": "AC4cac489c46197c9ebc91c840120a4dee", "date_created": "2020-11-25T09:36:42Z", "type": "Full", "subresource_uris": {"addresses": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/Addresses.json", "conferences": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/Conferences.json", "signing_keys": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/SigningKeys.json", "transcriptions": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/Transcriptions.json", "connect_apps": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/ConnectApps.json", "sip": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/SIP.json", "authorized_connect_apps": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AuthorizedConnectApps.json", "usage": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/Usage.json", "keys": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/Keys.json", "applications": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/Applications.json", "recordings": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/Recordings.json", "short_codes": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/SMS/ShortCodes.json", "calls": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/Calls.json", "notifications": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/Notifications.json", "incoming_phone_numbers": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/IncomingPhoneNumbers.json", "queues": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/Queues.json", "messages": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/Messages.json", "outgoing_caller_ids": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/OutgoingCallerIds.json", "available_phone_numbers": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers.json", "balance": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/Balance.json"}}, "emitted_at": 1669128692067}

airbyte-integrations/connectors/source-twilio/setup.py

@@ -7,8 +7,8 @@
 
 MAIN_REQUIREMENTS = [
     "airbyte-cdk",
-    "pendulum~=2.1",
-    "requests~=2.25",
+    "pendulum",
+    "requests",
 ]
 
 TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock", "requests_mock", "freezegun"]