Skip to content

Commit

Permalink
Source Marketo: set high test strictness level (#21947)
Browse files Browse the repository at this point in the history
* #19055 source marketo: set high test strictness level

* #19055 source marketo: increase incremental test timeout

* #19055 source Marketo: fix stream schemas

* #19055 source marketoL fix unit test

* Updated expected records

* Updated expected records

* auto-bump connector version

---------

Co-authored-by: Denys Davydov <davydov.den18@gmail.com>
Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
  • Loading branch information
3 people committed Jan 31, 2023
1 parent e4f4412 commit 92c1a23
Show file tree
Hide file tree
Showing 11 changed files with 315 additions and 167 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -1023,7 +1023,7 @@
- name: Marketo
sourceDefinitionId: 9e0556f4-69df-4522-a3fb-03264d36b348
dockerRepository: airbyte/source-marketo
dockerImageTag: 0.1.11
dockerImageTag: 1.0.0
documentationUrl: https://docs.airbyte.com/integrations/sources/marketo
icon: marketo.svg
sourceType: api
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8141,7 +8141,7 @@
supportsNormalization: false
supportsDBT: false
supported_destination_sync_modes: []
- dockerImage: "airbyte/source-marketo:0.1.11"
- dockerImage: "airbyte/source-marketo:1.0.0"
spec:
documentationUrl: "https://docs.airbyte.com/integrations/sources/marketo"
connectionSpecification:
Expand Down
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/source-marketo/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -34,5 +34,5 @@ COPY source_marketo ./source_marketo
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]

LABEL io.airbyte.version=0.1.11
LABEL io.airbyte.version=1.0.0
LABEL io.airbyte.name=airbyte/source-marketo
Original file line number Diff line number Diff line change
@@ -1,29 +1,82 @@
# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference)
# for more information about how to configure these tests
connector_image: airbyte/source-marketo:dev
tests:
test_strictness_level: high
acceptance_tests:
spec:
- spec_path: "source_marketo/spec.json"
tests:
- spec_path: "source_marketo/spec.json"
connection:
- config_path: "secrets/config.json"
status: "succeed"
- config_path: "integration_tests/invalid_config.json"
status: "failed"
tests:
- config_path: "secrets/config.json"
status: "succeed"
- config_path: "integration_tests/invalid_config.json"
status: "failed"
discovery:
- config_path: "secrets/config.json"
tests:
- config_path: "secrets/config.json"
basic_read:
- config_path: "secrets/config.json"
configured_catalog_path: "integration_tests/configured_catalog.json"
empty_streams: ["lists", "campaigns"]
timeout_seconds: 3600
expect_records:
path: "integration_tests/expected_records.jsonl"
tests:
- config_path: "secrets/config.json"
expect_records:
path: "integration_tests/expected_records.jsonl"
empty_streams:
- name: "activities_update_opportunity"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_change_segment"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_document_interacted_with"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_call_webhook"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_push_leadto_marketo"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_change_revenue_stage"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_dialogue_goal_reached"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_removefrom_opportunity"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_received_forwardto_friend_email"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_open_sales_email"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_share_content"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_sales_email_bounced"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_dialogue_engaged"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_dialogue_appointment_scheduled"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_click_sales_email"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_sent_forwardto_friend_email"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_click_link"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_send_sales_email"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_change_revenue_stage_manually"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_receive_sales_email"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_replyto_sales_email"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_addto_opportunity"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_open_email"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
# 52 streams, most of them use BULK API therefore it takes much time to run a sync
timeout_seconds: 9000
incremental:
- config_path: "secrets/config.json"
configured_catalog_path: "integration_tests/incremental_catalog.json"
future_state_path: "integration_tests/abnormal_state.json"
timeout_seconds: 3600
tests:
- config_path: "secrets/config.json"
configured_catalog_path: "integration_tests/incremental_catalog.json"
future_state:
future_state_path: "integration_tests/abnormal_state.json"
timeout_seconds: 3600
full_refresh:
- config_path: "secrets/config.json"
configured_catalog_path: "integration_tests/configured_catalog.json"
timeout_seconds: 3600
tests:
- config_path: "secrets/config.json"
configured_catalog_path: "integration_tests/configured_catalog.json"

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -4,65 +4,84 @@
"stream": {
"name": "programs",
"json_schema": {},
"supported_sync_modes": ["incremental"],
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": []
"default_cursor_field": ["updatedAt"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "append"
"sync_mode": "full_refresh",
"destination_sync_mode": "append",
"cursor_field": ["updatedAt"],
"primary_key": [["id"]]
},
{
"stream": {
"name": "campaigns",
"json_schema": {},
"supported_sync_modes": ["incremental"],
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": []
"default_cursor_field": ["createdAt"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "append"
"sync_mode": "full_refresh",
"destination_sync_mode": "append",
"cursor_field": ["createdAt"],
"primary_key": [["id"]]
},
{
"stream": {
"name": "lists",
"json_schema": {},
"supported_sync_modes": ["incremental"],
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": []
"default_cursor_field": ["createdAt"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "append"
"sync_mode": "full_refresh",
"destination_sync_mode": "append",
"cursor_field": ["createdAt"],
"primary_key": [["id"]]
},
{
"stream": {
"name": "leads",
"json_schema": {},
"supported_sync_modes": ["incremental"],
"source_defined_cursor": true,
"default_cursor_field": []
"supported_sync_modes": ["full_refresh", "incremental"],
"default_cursor_field": ["updatedAt"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "append"
"sync_mode": "full_refresh",
"destination_sync_mode": "append",
"cursor_field": ["updatedAt"],
"primary_key": [["id"]]
},
{
"stream": {
"name": "activity_types",
"json_schema": {},
"supported_sync_modes": ["full_refresh"]
"supported_sync_modes": ["full_refresh"],
"source_defined_cursor": false,
"default_cursor_field": null,
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "append"
"destination_sync_mode": "append",
"cursor_field": null,
"primary_key": [["id"]]
},
{
"stream": {
"name": "activities_visit_webpage",
"json_schema": {},
"supported_sync_modes": ["incremental"],
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": []
"default_cursor_field": ["activityDate"],
"source_defined_primary_key": [["marketoGUID"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "append"
"sync_mode": "full_refresh",
"destination_sync_mode": "append",
"cursor_field": ["activityDate"],
"primary_key": [["marketoGUID"]]
}
]
}

0 comments on commit 92c1a23

Please sign in to comment.