Source Close.com: migration to low-code (#18968)

* Source Close.com: migration to low-code * Source Close.com: bump version; update changelog * Source Close.com: documentation structure standartization * Source Close.com: making connector to be prepared to beta * Source Close.com: migrate SAT config to new version * Source Close.com: unit tests for custom components Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
airbytehq · Dec 7, 2022 · ab44197 · ab44197
1 parent 3532931
commit ab44197
Show file tree

Hide file tree

Showing 21 changed files with 1,344 additions and 71 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -255,7 +255,7 @@
 - name: Close.com
   sourceDefinitionId: dfffecb7-9a13-43e9-acdc-b92af7997ca9
   dockerRepository: airbyte/source-close-com
-  dockerImageTag: 0.1.0
+  dockerImageTag: 0.2.0
   documentationUrl: https://docs.airbyte.com/integrations/sources/close-com
   icon: close.svg
   sourceType: api
@@ -267,7 +267,7 @@
   documentationUrl: https://docs.airbyte.com/integrations/sources/coingecko-coins
   icon: coingeckocoins.svg
   sourceType: api
-  releaseStage: alpha
+  releaseStage: beta
 - name: Cockroachdb
   sourceDefinitionId: 9fa5862c-da7c-11eb-8d19-0242ac130003
   dockerRepository: airbyte/source-cockroachdb

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -2422,7 +2422,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-close-com:0.1.0"
+- dockerImage: "airbyte/source-close-com:0.2.0"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/close-com"
     connectionSpecification:
@@ -2431,7 +2431,7 @@
       type: "object"
       required:
       - "api_key"
-      additionalProperties: false
+      additionalProperties: true
       properties:
         api_key:
           type: "string"

diff --git a/airbyte-integrations/connectors/source-close-com/Dockerfile b/airbyte-integrations/connectors/source-close-com/Dockerfile
@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.0
+LABEL io.airbyte.version=0.2.0
 LABEL io.airbyte.name=airbyte/source-close-com
diff --git a/airbyte-integrations/connectors/source-close-com/acceptance-test-config.yml b/airbyte-integrations/connectors/source-close-com/acceptance-test-config.yml
@@ -1,27 +1,52 @@
 # See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference)
 # for more information about how to configure these tests
 connector_image: airbyte/source-close-com:dev
-tests:
+acceptance_tests:
   spec:
-    - spec_path: "source_close_com/spec.json"
+    tests:
+      - spec_path: "source_close_com/spec.json"
   connection:
-    - config_path: "secrets/config.json"
-      status: "succeed"
-    - config_path: "integration_tests/invalid_config.json"
-      status: "failed"
+    tests:
+      - config_path: "secrets/config.json"
+        status: "succeed"
+      - config_path: "integration_tests/invalid_config.json"
+        status: "failed"
   discovery:
-    - config_path: "secrets/config.json"
+    tests:
+      - config_path: "secrets/config.json"
+        # Some field types was changed for streams.
+        # custom_activities: id field was added
+        # custom_email_connected_accounts: imap field was changed to object
+        # smart_views: imap field was changed to object
+        backward_compatibility_tests_config:
+          disable_for_version: "0.1.0"
   basic_read:
-    - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog.json"
-      empty_streams: ["meeting_activities", "missed_call_tasks", "email_followup_tasks", "answered_detached_call_tasks",
-                      "incoming_sms_tasks", "zoom_connected_accounts", "custom_email_connected_accounts",
-                      "google_connected_accounts", "send_as", "email_sequences", "dialer", "email_bulk_actions",
-                      "sequence_subscription_bulk_actions", "delete_bulk_actions"]
+    tests:
+      - config_path: "secrets/config.json"
+        configured_catalog_path: "integration_tests/configured_catalog.json"
+        expect_records:
+          path: "integration_tests/expected_records.txt"
+          extra_fields: no
+          exact_order: no
+          extra_records: yes
+        empty_streams:
+          - name: missed_call_tasks
+            bypass_reason: "unable to populate"
+          - name: answered_detached_call_tasks
+            bypass_reason: "unable to populate"
+          - name: incoming_sms_tasks
+            bypass_reason: "unable to populate"
+          - name: send_as
+            bypass_reason: "unable to populate"
+          - name: voicemail_tasks
+            bypass_reason: "unable to populate"
   incremental:
-    - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog.json"
-      future_state_path: "integration_tests/abnormal_state.json"
+    tests:
+      - config_path: "secrets/config.json"
+        configured_catalog_path: "integration_tests/configured_catalog.json"
+        future_state:
+          future_state_path: "integration_tests/abnormal_state.json"
   full_refresh:
-    - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog.json"
+    tests:
+      - config_path: "secrets/config.json"
+        configured_catalog_path: "integration_tests/configured_catalog.json"
diff --git a/airbyte-integrations/connectors/source-close-com/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-close-com/acceptance-test-docker.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env sh
 
 # Build latest connector image
-docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2)
+docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-)
 
 # Pull latest acctest image
 docker pull airbyte/source-acceptance-test:latest

diff --git a/airbyte-integrations/connectors/source-close-com/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-close-com/integration_tests/abnormal_state.json
@@ -30,7 +30,7 @@
     "date_created": "2050-07-13T11:39:05.012000+00:00"
   },
   "leads": {
-    "date_updated": "2050-07-16T00:00:12.701000"
+    "date_updated": "2050-07-16T00:00:12.701000+00:00"
   },
   "lead_tasks": {
     "date_created": "2050-07-13T11:39:03.520000+00:00"

diff --git a/airbyte-integrations/connectors/source-close-com/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-close-com/integration_tests/expected_records.txt
diff --git a/airbyte-integrations/connectors/source-close-com/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-close-com/integration_tests/invalid_config.json
@@ -1,4 +1,4 @@
 {
   "api_key": "api_wrong",
-  "start_date": "2021-01-01"
+  "start_date": "2021-01-01T00:00:00.000000+00:00"
 }
diff --git a/airbyte-integrations/connectors/source-close-com/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-close-com/integration_tests/sample_config.json
@@ -1,4 +1,4 @@
 {
   "api_key": "api_1234567890",
-  "start_date": "2021-01-01"
+  "start_date": "2021-01-01T00:00:00.000000+00:00"
 }
diff --git a/airbyte-integrations/connectors/source-close-com/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-close-com/integration_tests/sample_state.json
@@ -1,34 +1,34 @@
 {
   "created_activities": {
-    "date_created": "2021-01-01"
+    "date_created": "2021-01-01T00:00:00.000000+00:00"
   },
   "opportunity_status_change_activities": {},
   "note_activities": {
-    "date_created": "2021-01-01"
+    "date_created": "2021-01-01T00:00:00.000000+00:00"
   },
   "meeting_activities": {},
   "call_activities": {
-    "date_created": "2021-01-01"
+    "date_created": "2021-01-01T00:00:00.000000+00:00"
   },
   "email_activities": {
-    "date_created": "2021-01-01"
+    "date_created": "2021-01-01T00:00:00.000000+00:00"
   },
   "email_thread_activities": {
-    "date_created": "2021-01-01"
+    "date_created": "2021-01-01T00:00:00.000000+00:00"
   },
   "lead_status_change_activities": {
-    "date_created": "2021-01-01"
+    "date_created": "2021-01-01T00:00:00.000000+00:00"
   },
   "sms_activities": {
-    "date_created": "2021-01-01"
+    "date_created": "2021-01-01T00:00:00.000000+00:00"
   },
   "task_completed_activities": {},
   "leads": {},
   "lead_tasks": {
-    "date_created": "2021-01-01"
+    "date_created": "2021-01-01T00:00:00.000000+00:00"
   },
   "incoming_email_tasks": {
-    "date_created": "2021-01-01"
+    "date_created": "2021-01-01T00:00:00.000000+00:00"
   },
   "email_followup_tasks": {
     "date_created": "2050-07-13T11:39:03.520000+00:00"
@@ -49,9 +49,9 @@
     "date_created": "2050-07-13T11:39:03.520000+00:00"
   },
   "events": {
-    "date_updated": "2021-01-01"
+    "date_updated": "2021-01-01T00:00:00.000000+00:00"
   },
   "opportunities": {
-    "date_updated": "2021-01-01"
+    "date_updated": "2021-01-01T00:00:00.000000+00:00"
   }
 }
diff --git a/airbyte-integrations/connectors/source-close-com/setup.py b/airbyte-integrations/connectors/source-close-com/setup.py
@@ -5,9 +5,7 @@
 
 from setuptools import find_packages, setup
 
-MAIN_REQUIREMENTS = [
-    "airbyte-cdk",
-]
+MAIN_REQUIREMENTS = ["airbyte-cdk", "pendulum", "requests"]
 
 TEST_REQUIREMENTS = [
     "pytest~=6.1",

diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/__init__.py b/airbyte-integrations/connectors/source-close-com/source_close_com/__init__.py
@@ -21,6 +21,7 @@
 # SOFTWARE.
 
 
-from .source import SourceCloseCom
+from .datetime_slicer import CustomDatetimeStreamSlicer
+from .source_lc import SourceCloseCom
 
-__all__ = ["SourceCloseCom"]
+__all__ = ["SourceCloseCom", "CustomDatetimeStreamSlicer"]