Source Twilio: adopt best practices (#14000)

* #1946 Source twilio: aopt best practices - tune tests * #1946 add expected_records to acceptance-test-config.yml * #1946 source twilio - upd schema and changelog * #1946 fix expected_records * #1946 source twilio: rm alerts from expected records as they expire in 30 days * #1946 source twilio: bump version
airbytehq · Jun 27, 2022 · 501a1c3 · 501a1c3
1 parent 9828246
commit 501a1c3
Show file tree

Hide file tree

Showing 10 changed files with 608 additions and 25 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -961,7 +961,7 @@
 - name: Twilio
   sourceDefinitionId: b9dc6155-672e-42ea-b10d-9f1f1fb95ab1
   dockerRepository: airbyte/source-twilio
-  dockerImageTag: 0.1.5
+  dockerImageTag: 0.1.6
   documentationUrl: https://docs.airbyte.io/integrations/sources/twilio
   icon: twilio.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -9299,7 +9299,7 @@
         oauthFlowOutputParameters:
         - - "token"
         - - "key"
-- dockerImage: "airbyte/source-twilio:0.1.5"
+- dockerImage: "airbyte/source-twilio:0.1.6"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/twilio"
     connectionSpecification:

diff --git a/airbyte-integrations/connectors/source-twilio/Dockerfile b/airbyte-integrations/connectors/source-twilio/Dockerfile
@@ -4,13 +4,13 @@ FROM python:3.9-slim
 RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /airbyte/integration_code
-COPY source_twilio ./source_twilio
-COPY main.py ./
 COPY setup.py ./
 RUN pip install .
+COPY source_twilio ./source_twilio
+COPY main.py ./
 
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.5
+LABEL io.airbyte.version=0.1.6
 LABEL io.airbyte.name=airbyte/source-twilio
diff --git a/airbyte-integrations/connectors/source-twilio/acceptance-test-config.yml b/airbyte-integrations/connectors/source-twilio/acceptance-test-config.yml
@@ -13,6 +13,8 @@ tests:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/no_empty_streams_catalog.json"
       empty_streams: ["message_media", "conferences"]
+      expect_records:
+        path: "integration_tests/expected_records.txt"
   incremental:
     - config_path: "secrets/config.json"
       # usage records stream produces and error if cursor date gte than current date

diff --git a/airbyte-integrations/connectors/source-twilio/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-twilio/integration_tests/expected_records.txt
diff --git a/...yte-integrations/connectors/source-twilio/integration_tests/no_empty_streams_catalog.json b/...yte-integrations/connectors/source-twilio/integration_tests/no_empty_streams_catalog.json
@@ -188,6 +188,18 @@
       },
       "sync_mode": "full_refresh",
       "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "alerts",
+        "json_schema": {},
+        "supported_sync_modes": ["incremental", "full_refresh"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["date_updated"]
+      },
+      "sync_mode": "incremental",
+      "cursor_field": ["date_updated"],
+      "destination_sync_mode": "append"
     }
   ]
 }
diff --git a/airbyte-integrations/connectors/source-twilio/setup.py b/airbyte-integrations/connectors/source-twilio/setup.py
@@ -11,9 +11,7 @@
     "requests~=2.25",
 ]
 
-TEST_REQUIREMENTS = [
-    "pytest~=6.1",
-]
+TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock"]
 
 setup(
     name="source_twilio",

diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/recordings.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/recordings.json
@@ -54,6 +54,9 @@
     "uri": {
       "type": ["null", "string"]
     },
+    "media_url": {
+       "type": "string"
+    },
     "encryption_details": {
       "properties": {
         "type": {

diff --git a/airbyte-integrations/connectors/source-twilio/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-twilio/unit_tests/unit_test.py
@@ -2,6 +2,39 @@
 # Copyright (c) 2022 Airbyte, Inc., all rights reserved.
 #
 
+from unittest.mock import Mock
 
-def test_example_method():
-    assert True
+import pytest
+import requests
+from source_twilio.source import SourceTwilio
+
+
+@pytest.fixture
+def config():
+    return {"account_sid": "airbyte.io", "auth_token": "secret", "start_date": "2022-01-01T00:00:00Z"}
+
+
+@pytest.mark.parametrize(
+    "exception, expected_error_msg",
+    (
+        (
+            ConnectionError("Connection aborted"),
+            "Unable to connect to Twilio API with the provided credentials - ConnectionError('Connection aborted')",
+        ),
+        (
+            TimeoutError("Socket timed out"),
+            "Unable to connect to Twilio API with the provided credentials - TimeoutError('Socket timed out')",
+        ),
+        (
+            requests.exceptions.HTTPError("401 Client Error: Unauthorized for url: https://api.twilio.com/"),
+            "Unable to connect to Twilio API with the provided credentials - "
+            "HTTPError('401 Client Error: Unauthorized for url: https://api.twilio.com/')",
+        ),
+    ),
+)
+def test_check_connection_handles_exceptions(mocker, config, exception, expected_error_msg):
+    mocker.patch.object(requests.Session, "send", Mock(side_effect=exception))
+    source = SourceTwilio()
+    status_ok, error = source.check_connection(logger=None, config=config)
+    assert not status_ok
+    assert error == expected_error_msg
diff --git a/docs/integrations/sources/twilio.md b/docs/integrations/sources/twilio.md
@@ -33,13 +33,13 @@ Several output streams are available from this source:
 
 ### Features
 
-| Feature | Supported? |
-| :--- | :--- |
-| Full Refresh Sync | Yes |
-| Incremental Sync | Yes |
-| Replicate Incremental Deletes | No |
-| SSL connection | Yes |
-| Namespaces | No |
+| Feature                       | Supported? |
+|:------------------------------|:-----------|
+| Full Refresh Sync             | Yes        |
+| Incremental Sync              | Yes        |
+| Replicate Incremental Deletes | No         |
+| SSL connection                | Yes        |
+| Namespaces                    | No         |
 
 ### Performance considerations
 
@@ -64,12 +64,13 @@ See [docs](https://www.twilio.com/docs/iam/api) for more details.
 
 ## Changelog
 
-| Version | Date | Pull Request | Subject |
-| :--- | :--- | :--- | :--- |
-| 0.1.5 | 2022-06-22 | [13896](https://github.com/airbytehq/airbyte/pull/13896) | Add lookback window parameters to fetch messages with a rolling window and catch status updates |
-| 0.1.4 | 2022-04-22 | [12157](https://github.com/airbytehq/airbyte/pull/12157) | Use Retry-After header for backoff |
-| 0.1.3 | 2022-04-20 | [12183](https://github.com/airbytehq/airbyte/pull/12183) | Add new subresource on the call stream + declare a valid primary key for conference_participants stream |
-| 0.1.2 | 2021-12-23 | [9092](https://github.com/airbytehq/airbyte/pull/9092) | Correct specification doc URL |
-| 0.1.1 | 2021-10-18 | [7034](https://github.com/airbytehq/airbyte/pull/7034) | Update schemas and transform data types according to the API schema |
-| 0.1.0 | 2021-07-02 | [4070](https://github.com/airbytehq/airbyte/pull/4070) | Native Twilio connector implemented |
+| Version | Date        | Pull Request                                             | Subject                                                                                                 |
+|:--------|:------------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------|
+| 0.1.6   | 2022-06-22  | [14000](https://github.com/airbytehq/airbyte/pull/14000) | Update Records stream schema and align tests with connectors' best practices                            |
+| 0.1.5   | 2022-06-22  | [13896](https://github.com/airbytehq/airbyte/pull/13896) | Add lookback window parameters to fetch messages with a rolling window and catch status updates         |
+| 0.1.4   | 2022-04-22  | [12157](https://github.com/airbytehq/airbyte/pull/12157) | Use Retry-After header for backoff                                                                      |
+| 0.1.3   | 2022-04-20  | [12183](https://github.com/airbytehq/airbyte/pull/12183) | Add new subresource on the call stream + declare a valid primary key for conference_participants stream |
+| 0.1.2   | 2021-12-23  | [9092](https://github.com/airbytehq/airbyte/pull/9092)   | Correct specification doc URL                                                                           |
+| 0.1.1   | 2021-10-18  | [7034](https://github.com/airbytehq/airbyte/pull/7034)   | Update schemas and transform data types according to the API schema                                     |
+| 0.1.0   | 2021-07-02  | [4070](https://github.com/airbytehq/airbyte/pull/4070)   | Native Twilio connector implemented                                                                     |