From 3b8f6e86f4723bab8604dc3cf9b98a2a7b8dd891 Mon Sep 17 00:00:00 2001 From: "oleh.zorenko" <19872253+Zirochkaa@users.noreply.github.com> Date: Mon, 30 Aug 2021 23:15:24 +0300 Subject: [PATCH] Source GitHub: use new repo for SAT tests (#5726) * 5721 Use new repo for GitHub SAT tests Use new `airbytehq/integration-test` repo. Remove unneeded files. * Update docs * Remove version bumping --- .../source-github/acceptance-test-config.yml | 35 ++--- .../integration_tests/abnormal_state.json | 22 +-- .../incremental_configured_catalog.json | 95 ------------ .../integration_tests/invalid_config.json | 2 +- .../integration_tests/sample_state.json | 22 +-- .../small_streams_configured_catalog.json | 138 ------------------ .../too_large_streams_configured_catalog.json | 105 ------------- 7 files changed, 37 insertions(+), 382 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-github/integration_tests/incremental_configured_catalog.json delete mode 100644 airbyte-integrations/connectors/source-github/integration_tests/small_streams_configured_catalog.json delete mode 100644 airbyte-integrations/connectors/source-github/integration_tests/too_large_streams_configured_catalog.json diff --git a/airbyte-integrations/connectors/source-github/acceptance-test-config.yml b/airbyte-integrations/connectors/source-github/acceptance-test-config.yml index 901d809bf8334..89699dd07ba18 100644 --- a/airbyte-integrations/connectors/source-github/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-github/acceptance-test-config.yml @@ -11,30 +11,23 @@ tests: - config_path: "secrets/config.json" basic_read: - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/small_streams_configured_catalog.json" - timeout_seconds: 3600 - # Below streams have too much records and because of it we are getting timeouts and hitting rate limits. - # That's why we turned them off. - # - config_path: "secrets/config.json" - # configured_catalog_path: "integration_tests/too_large_streams_configured_catalog.json" - # TODO: Should be fixed after we create new repositories for testing (issue #4750) + configured_catalog_path: "integration_tests/configured_catalog.json" incremental: - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/incremental_configured_catalog.json" + configured_catalog_path: "integration_tests/configured_catalog.json" future_state_path: "integration_tests/abnormal_state.json" cursor_paths: - issues: ["airbytehq/airbyte", "updated_at"] - comments: ["airbytehq/airbyte", "updated_at"] - commits: ["airbytehq/airbyte", "created_at"] - commit_comments: ["airbytehq/airbyte", "updated_at"] - issue_milestones: ["airbytehq/airbyte", "updated_at"] - projects: ["airbytehq/airbyte", "updated_at"] - releases: ["airbytehq/airbyte", "created_at"] + events: ["airbytehq/integration-test", "created_at"] + comments: ["airbytehq/integration-test", "updated_at"] + pull_requests: ["airbytehq/integration-test", "updated_at"] + commit_comments: ["airbytehq/integration-test", "updated_at"] + issue_milestones: ["airbytehq/integration-test", "updated_at"] + commits: ["airbytehq/integration-test", "created_at"] + stargazers: ["airbytehq/integration-test", "starred_at"] + projects: ["airbytehq/integration-test", "updated_at"] + issues: ["airbytehq/integration-test", "updated_at"] + issue_events: ["airbytehq/integration-test", "created_at"] + releases: ["airbytehq/integration-test", "created_at"] full_refresh: - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/small_streams_configured_catalog.json" - timeout_seconds: 3600 - # Below streams have too much records and because of it we are getting timeouts and hitting rate limits. - # That's why we turned them off. -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/too_large_streams_configured_catalog.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-github/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-github/integration_tests/abnormal_state.json index 65eb95569a94c..91300c4991090 100644 --- a/airbyte-integrations/connectors/source-github/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-github/integration_tests/abnormal_state.json @@ -1,57 +1,57 @@ { "commit_comments": { - "airbytehq/airbyte": { + "airbytehq/integration-test": { "updated_at": "2121-04-30T20:36:17Z" } }, "projects": { - "airbytehq/airbyte": { + "airbytehq/integration-test": { "updated_at": "2121-06-28T17:24:51Z" } }, "releases": { - "airbytehq/airbyte": { + "airbytehq/integration-test": { "created_at": "2121-06-23T23:57:07Z" } }, "issue_milestones": { - "airbytehq/airbyte": { + "airbytehq/integration-test": { "updated_at": "2121-06-25T22:28:33Z" } }, "issues": { - "airbytehq/airbyte": { + "airbytehq/integration-test": { "updated_at": "2121-06-30T06:44:42Z" } }, "comments": { - "airbytehq/airbyte": { + "airbytehq/integration-test": { "updated_at": "2121-06-30T10:22:10Z" } }, "commits": { - "airbytehq/airbyte": { + "airbytehq/integration-test": { "created_at": "2121-06-30T10:04:41Z" } }, "stargazers": { - "airbytehq/airbyte": { + "airbytehq/integration-test": { "starred_at": "2121-06-29T02:04:57Z" } }, "events": { - "airbytehq/airbyte": { + "airbytehq/integration-test": { "created_at": "2121-06-29T03:44:45Z" } }, "issue_events": { - "airbytehq/airbyte": { + "airbytehq/integration-test": { "created_at": "2121-06-29T01:49:42Z" } }, "pull_requests": { - "airbytehq/airbyte": { + "airbytehq/integration-test": { "updated_at": "2121-06-28T23:36:35Z" } } diff --git a/airbyte-integrations/connectors/source-github/integration_tests/incremental_configured_catalog.json b/airbyte-integrations/connectors/source-github/integration_tests/incremental_configured_catalog.json deleted file mode 100644 index 37272ff421dc2..0000000000000 --- a/airbyte-integrations/connectors/source-github/integration_tests/incremental_configured_catalog.json +++ /dev/null @@ -1,95 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "issues", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "default_cursor_field": ["updated_at"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["updated_at"] - }, - { - "stream": { - "name": "comments", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "default_cursor_field": ["updated_at"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["updated_at"] - }, - { - "stream": { - "name": "commits", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "default_cursor_field": ["created_at"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["created_at"] - }, - { - "stream": { - "name": "commit_comments", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "default_cursor_field": ["updated_at"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["updated_at"] - }, - { - "stream": { - "name": "issue_milestones", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "default_cursor_field": ["updated_at"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["updated_at"] - }, - { - "stream": { - "name": "projects", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "default_cursor_field": ["updated_at"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["updated_at"] - }, - { - "stream": { - "name": "releases", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "default_cursor_field": ["created_at"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["created_at"] - } - ] -} diff --git a/airbyte-integrations/connectors/source-github/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-github/integration_tests/invalid_config.json index 81d882dcf2f18..59183b9d2c9a0 100644 --- a/airbyte-integrations/connectors/source-github/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-github/integration_tests/invalid_config.json @@ -1,5 +1,5 @@ { "access_token": "WRONG KEY", - "repository": "airbytehq/airbyte", + "repository": "airbytehq/integration-test", "start_date": "2021-06-30T11:30:03Z" } diff --git a/airbyte-integrations/connectors/source-github/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-github/integration_tests/sample_state.json index 27a0bdfed265f..068a99d1f7172 100644 --- a/airbyte-integrations/connectors/source-github/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-github/integration_tests/sample_state.json @@ -1,56 +1,56 @@ { "commit_comments": { - "airbytehq/airbyte": { + "airbytehq/integration-test": { "updated_at": "2021-04-30T20:36:17Z" } }, "projects": { - "airbytehq/airbyte": { + "airbytehq/integration-test": { "updated_at": "2021-06-28T17:24:51Z" } }, "stargazers": { - "airbytehq/airbyte": { + "airbytehq/integration-test": { "starred_at": "2021-06-29T02:04:57Z" } }, "events": { - "airbytehq/airbyte": { + "airbytehq/integration-test": { "created_at": "2021-06-29T03:44:45Z" } }, "issue_events": { - "airbytehq/airbyte": { + "airbytehq/integration-test": { "created_at": "2021-06-29T01:49:42Z" } }, "releases": { - "airbytehq/airbyte": { + "airbytehq/integration-test": { "created_at": "2021-06-23T23:57:07Z" } }, "pull_requests": { - "airbytehq/airbyte": { + "airbytehq/integration-test": { "updated_at": "2021-06-28T23:36:35Z" } }, "issue_milestones": { - "airbytehq/airbyte": { + "airbytehq/integration-test": { "updated_at": "2021-06-25T22:28:33Z" } }, "issues": { - "airbytehq/airbyte": { + "airbytehq/integration-test": { "updated_at": "2021-06-30T11:32:49Z" } }, "comments": { - "airbytehq/airbyte": { + "airbytehq/integration-test": { "updated_at": "2021-06-30T10:22:10Z" } }, "commits": { - "airbytehq/airbyte": { + "airbytehq/integration-test": { "created_at": "2021-06-30T10:04:41Z" } } diff --git a/airbyte-integrations/connectors/source-github/integration_tests/small_streams_configured_catalog.json b/airbyte-integrations/connectors/source-github/integration_tests/small_streams_configured_catalog.json deleted file mode 100644 index cbf6fc07e1fc6..0000000000000 --- a/airbyte-integrations/connectors/source-github/integration_tests/small_streams_configured_catalog.json +++ /dev/null @@ -1,138 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "comments", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "default_cursor_field": ["updated_at"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["updated_at"] - }, - { - "stream": { - "name": "commit_comments", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "default_cursor_field": ["updated_at"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["updated_at"] - }, - { - "stream": { - "name": "issue_milestones", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "default_cursor_field": ["updated_at"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["updated_at"] - }, - { - "stream": { - "name": "stargazers", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "default_cursor_field": ["starred_at"], - "source_defined_primary_key": [["user_id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["starred_at"] - }, - { - "stream": { - "name": "collaborators", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "pull_requests", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "default_cursor_field": ["updated_at"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["updated_at"] - }, - { - "stream": { - "name": "teams", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "projects", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "default_cursor_field": ["updated_at"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["updated_at"] - }, - { - "stream": { - "name": "issues", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "default_cursor_field": ["updated_at"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["updated_at"] - }, - { - "stream": { - "name": "issue_labels", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "releases", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "default_cursor_field": ["created_at"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["created_at"] - } - ] -} diff --git a/airbyte-integrations/connectors/source-github/integration_tests/too_large_streams_configured_catalog.json b/airbyte-integrations/connectors/source-github/integration_tests/too_large_streams_configured_catalog.json deleted file mode 100644 index 22bbab21ec76f..0000000000000 --- a/airbyte-integrations/connectors/source-github/integration_tests/too_large_streams_configured_catalog.json +++ /dev/null @@ -1,105 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "reviews", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "events", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "default_cursor_field": ["created_at"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["created_at"] - }, - { - "stream": { - "name": "comments", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "default_cursor_field": ["updated_at"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["updated_at"] - }, - { - "stream": { - "name": "pull_requests", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "default_cursor_field": ["updated_at"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["updated_at"] - }, - { - "stream": { - "name": "commits", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "default_cursor_field": ["created_at"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["created_at"] - }, - { - "stream": { - "name": "stargazers", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "default_cursor_field": ["starred_at"], - "source_defined_primary_key": [["user_id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["starred_at"] - }, - { - "stream": { - "name": "issues", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "default_cursor_field": ["updated_at"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["updated_at"] - }, - { - "stream": { - "name": "issue_events", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "default_cursor_field": ["created_at"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["created_at"] - } - ] -}