Source Github: Raise Error if no organizations or repos are available (…

…#21084) * Source Github: Raise Error if no organizations or repos are available * Source GitHub: Update docs; bump version * Source GitHub: Add test * Source GitHub: Exclude empty streams * auto-bump connector version Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
airbytehq · Jan 13, 2023 · 6403184 · 6403184
1 parent 06788f0
commit 6403184
Show file tree

Hide file tree

Showing 7 changed files with 74 additions and 52 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -561,7 +561,7 @@
 - name: GitHub
   sourceDefinitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e
   dockerRepository: airbyte/source-github
-  dockerImageTag: 0.3.10
+  dockerImageTag: 0.3.11
   documentationUrl: https://docs.airbyte.com/integrations/sources/github
   icon: github.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -4587,7 +4587,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-github:0.3.10"
+- dockerImage: "airbyte/source-github:0.3.11"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/github"
     connectionSpecification:

diff --git a/airbyte-integrations/connectors/source-github/Dockerfile b/airbyte-integrations/connectors/source-github/Dockerfile
@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.3.10
+LABEL io.airbyte.version=0.3.11
 LABEL io.airbyte.name=airbyte/source-github
diff --git a/airbyte-integrations/connectors/source-github/acceptance-test-config.yml b/airbyte-integrations/connectors/source-github/acceptance-test-config.yml
@@ -1,55 +1,65 @@
 connector_image: airbyte/source-github:dev
-tests:
+acceptance_tests:
   spec:
-    - spec_path: "source_github/spec.json"
+    tests:
+      - spec_path: "source_github/spec.json"
   connection:
-    - config_path: "secrets/config.json"
-      status: "succeed"
-    - config_path: "secrets/config_oauth.json"
-      status: "succeed"
-    - config_path: "integration_tests/invalid_config.json"
-      status: "failed"
+    tests:
+      - config_path: "secrets/config.json"
+        status: "succeed"
+      - config_path: "secrets/config_oauth.json"
+        status: "succeed"
+      - config_path: "integration_tests/invalid_config.json"
+        status: "failed"
   discovery:
-    - config_path: "secrets/config.json"
-    - config_path: "secrets/config_oauth.json"
+    tests:
+      - config_path: "secrets/config.json"
+      - config_path: "secrets/config_oauth.json"
   basic_read:
-    - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog.json"
+    tests:
+      - config_path: "secrets/config.json"
+        configured_catalog_path: "integration_tests/configured_catalog.json"
+        empty_streams:
+          - name: "events"
+            bypass_reason: "Only events created within the past 90 days can be showed"
   incremental:
-    - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog.json"
-      future_state_path: "integration_tests/abnormal_state.json"
-      cursor_paths:
-        comments: ["airbytehq/integration-test", "updated_at"]
-        commit_comment_reactions:
-          ["airbytehq/integration-test", "55538825", "created_at"]
-        commit_comments: ["airbytehq/integration-test", "updated_at"]
-        commits: ["airbytehq/integration-test", "master", "created_at"]
-        deployments: ["airbytehq/integration-test", "updated_at"]
-        events: ["airbytehq/integration-test", "created_at"]
-        issue_comment_reactions:
-          ["airbytehq/integration-test", "907296275", "created_at"]
-        issue_events: ["airbytehq/integration-test", "created_at"]
-        issue_milestones: ["airbytehq/integration-test", "updated_at"]
-        issue_reactions: ["airbytehq/integration-test", "11", "created_at"]
-        issues: ["airbytehq/integration-test", "updated_at"]
-        project_cards:
-          ["airbytehq/integration-test", "13167124", "17807006", "updated_at"]
-        project_columns:
-          ["airbytehq/integration-test", "13167124", "updated_at"]
-        projects: ["airbytehq/integration-test", "updated_at"]
-        pull_request_comment_reactions:
-          ["airbytehq/integration-test", "699253726", "created_at"]
-        pull_request_stats: ["airbytehq/integration-test", "updated_at"]
-        pull_requests: ["airbytehq/integration-test", "updated_at"]
-        releases: ["airbytehq/integration-test", "created_at"]
-        repositories: ["airbytehq", "updated_at"]
-        review_comments: ["airbytehq/integration-test", "updated_at"]
-        reviews: ["airbytehq/integration-test", "updated_at"]
-        stargazers: ["airbytehq/integration-test", "starred_at"]
-        workflow_runs: ["airbytehq/integration-test", "updated_at"]
-        workflows: ["airbytehq/integration-test", "updated_at"]
-        workflow_jobs: ["airbytehq/integration-test", "completed_at"]
+    tests:
+      - config_path: "secrets/config.json"
+        configured_catalog_path: "integration_tests/configured_catalog.json"
+        future_state:
+          future_state_path: "integration_tests/abnormal_state.json"
+        cursor_paths:
+          comments: ["airbytehq/integration-test", "updated_at"]
+          commit_comment_reactions:
+            ["airbytehq/integration-test", "55538825", "created_at"]
+          commit_comments: ["airbytehq/integration-test", "updated_at"]
+          commits: ["airbytehq/integration-test", "master", "created_at"]
+          deployments: ["airbytehq/integration-test", "updated_at"]
+          events: ["airbytehq/integration-test", "created_at"]
+          issue_comment_reactions:
+            ["airbytehq/integration-test", "907296275", "created_at"]
+          issue_events: ["airbytehq/integration-test", "created_at"]
+          issue_milestones: ["airbytehq/integration-test", "updated_at"]
+          issue_reactions: ["airbytehq/integration-test", "11", "created_at"]
+          issues: ["airbytehq/integration-test", "updated_at"]
+          project_cards:
+            ["airbytehq/integration-test", "13167124", "17807006", "updated_at"]
+          project_columns:
+            ["airbytehq/integration-test", "13167124", "updated_at"]
+          projects: ["airbytehq/integration-test", "updated_at"]
+          pull_request_comment_reactions:
+            ["airbytehq/integration-test", "699253726", "created_at"]
+          pull_request_stats: ["airbytehq/integration-test", "updated_at"]
+          pull_requests: ["airbytehq/integration-test", "updated_at"]
+          releases: ["airbytehq/integration-test", "created_at"]
+          repositories: ["airbytehq", "updated_at"]
+          review_comments: ["airbytehq/integration-test", "updated_at"]
+          reviews: ["airbytehq/integration-test", "updated_at"]
+          stargazers: ["airbytehq/integration-test", "starred_at"]
+          workflow_runs: ["airbytehq/integration-test", "updated_at"]
+          workflows: ["airbytehq/integration-test", "updated_at"]
+          workflow_jobs: ["airbytehq/integration-test", "completed_at"]
   full_refresh:
-    - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog.json"
+    tests:
+      - config_path: "secrets/config.json"
+        configured_catalog_path: "integration_tests/configured_catalog.json"
diff --git a/airbyte-integrations/connectors/source-github/source_github/source.py b/airbyte-integrations/connectors/source-github/source_github/source.py
@@ -176,6 +176,8 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) ->
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
         authenticator = self._get_authenticator(config)
         organizations, repositories = self._get_org_repositories(config=config, authenticator=authenticator)
+        if not any((organizations, repositories)):
+            raise Exception("No streams available. Please check permissions")
         page_size = config.get("page_size_for_large_streams", DEFAULT_PAGE_SIZE_FOR_LARGE_STREAM)
 
         organization_args = {"authenticator": authenticator, "organizations": organizations}

diff --git a/airbyte-integrations/connectors/source-github/unit_tests/test_source.py b/airbyte-integrations/connectors/source-github/unit_tests/test_source.py
@@ -127,3 +127,12 @@ def test_get_org_repositories():
 
     assert set(repositories) == {"airbytehq/integration-test", "docker/docker-py", "docker/compose"}
     assert set(organisations) == {"airbytehq", "docker"}
+
+
+def test_organization_or_repo_available():
+    SourceGithub._get_org_repositories = MagicMock(return_value=(False, False))
+    source = SourceGithub()
+    with pytest.raises(Exception) as exc_info:
+        config = {"access_token": "test_token", "repository": ""}
+        source.streams(config=config)
+    assert exc_info.value.args[0] == "No streams available. Please check permissions"
diff --git a/docs/integrations/sources/github.md b/docs/integrations/sources/github.md
@@ -162,7 +162,8 @@ The GitHub connector should not run into GitHub API limitations under normal usa
 ## Changelog
 
 | Version | Date       | Pull Request                                                                                                      | Subject                                                                                                                                                             |
-| :------ | :--------- | :---------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+|:--------|:-----------|:------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.3.11  | 2023-01-06 | [21084](https://github.com/airbytehq/airbyte/pull/21084)                                                          | Raise Error if no organizations or repos are available during read                                                                                                  |
 | 0.3.10  | 2022-12-15 | [20523](https://github.com/airbytehq/airbyte/pull/20523)                                                          | Revert changes from 0.3.9                                                                                                                                           |
 | 0.3.9   | 2022-12-14 | [19978](https://github.com/airbytehq/airbyte/pull/19978)                                                          | Update CDK dependency; move custom HTTPError handling into `AvailabilityStrategy` classes                                                                           |
 | 0.3.8   | 2022-11-10 | [19299](https://github.com/airbytehq/airbyte/pull/19299)                                                          | Fix events and workflow_runs datetimes                                                                                                                              |