Source Github: Repository field accept full URL (#25778)

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>

airbyte-integrations/connectors/source-github/Dockerfile

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.5.0
+LABEL io.airbyte.version=1.0.0
 LABEL io.airbyte.name=airbyte/source-github

airbyte-integrations/connectors/source-github/acceptance-test-config.yml

@@ -4,6 +4,8 @@ acceptance_tests:
   spec:
     tests:
     - spec_path: "source_github/spec.json"
+      backward_compatibility_tests_config:
+        disable_for_version: "0.5.0"
   connection:
     tests:
     - config_path: "secrets/config.json"

airbyte-integrations/connectors/source-github/metadata.yaml

@@ -5,7 +5,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e
-  dockerImageTag: 0.5.0
+  dockerImageTag: 1.0.0
   maxSecondsBetweenMessages: 5400
   dockerRepository: airbyte/source-github
   githubIssueLabel: source-github

airbyte-integrations/connectors/source-github/source_github/source.py

@@ -2,7 +2,6 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-import re
 from typing import Any, Dict, List, Mapping, Set, Tuple
 
 from airbyte_cdk import AirbyteLogger
@@ -57,23 +56,6 @@
 
 
 class SourceGithub(AbstractSource):
-    @staticmethod
-    def _is_repositories_config_valid(config_repositories: Set[str]) -> bool:
-        """
-        _is_repositories_config_valid validates that each repo config matches regex to highlight problem in provided config.
-        Valid examples: airbytehq/airbyte airbytehq/another-repo airbytehq/* airbytehq/airbyte
-        Args:
-            config_repositories: set of provided repositories
-        Returns:
-            True if config valid, False if it's not
-        """
-        pattern = re.compile(r"^(?:[\w.-]+/)+(?:\*|[\w.-]+)$")
-
-        for repo in config_repositories:
-            if not pattern.match(repo):
-                return False
-        return True
-
     @staticmethod
     def _get_and_prepare_repositories_config(config: Mapping[str, Any]) -> Set[str]:
         """
@@ -84,11 +66,6 @@ def _get_and_prepare_repositories_config(config: Mapping[str, Any]) -> Set[str]:
             set of provided repositories
         """
         config_repositories = set(filter(None, config["repository"].split(" ")))
-        # removing spaces
-        config_repositories = {repo.strip() for repo in config_repositories}
-        # removing redundant / in the end
-        config_repositories = {repo[:-1] if repo.endswith("/") else repo for repo in config_repositories}
-
         return config_repositories
 
     @staticmethod
@@ -100,14 +77,6 @@ def _get_org_repositories(config: Mapping[str, Any], authenticator: MultipleToke
             authenticator(MultipleTokenAuthenticator): authenticator object
         """
         config_repositories = SourceGithub._get_and_prepare_repositories_config(config)
-        if not SourceGithub._is_repositories_config_valid(config_repositories):
-            raise Exception(
-                f"You provided invalid format of repositories config: {' ' .join(config_repositories)}."
-                f" Valid examples: airbytehq/airbyte airbytehq/another-repo airbytehq/* airbytehq/airbyte"
-            )
-
-        if not config_repositories:
-            raise Exception("Field `repository` required to be provided for connect to Github API")
 
         repositories = set()
         organizations = set()

airbyte-integrations/connectors/source-github/source_github/spec.json

@@ -71,7 +71,8 @@
         "title": "GitHub Repositories",
         "description": "Space-delimited list of GitHub organizations/repositories, e.g. `airbytehq/airbyte` for single repository, `airbytehq/*` for get all repositories from organization and `airbytehq/airbyte airbytehq/another-repo` for multiple repositories.",
         "order": 2,
-        "pattern_descriptor": "org/repo1 org/repo2"
+        "pattern": "^([\\w.-]+/(\\*|[\\w.-]+(?<!\\.git))\\s+)*[\\w.-]+/(\\*|[\\w.-]+(?<!\\.git))$",
+        "pattern_descriptor": "org/repo org/another-repo org/*"
       },
       "branch": {
         "type": "string",

airbyte-integrations/connectors/source-github/unit_tests/test_source.py

@@ -14,6 +14,8 @@
 from source_github.source import SourceGithub
 from source_github.utils import MultipleTokenAuthenticatorWithRateLimiter
 
+from .utils import command_check
+
 
 def check_source(repo_line: str) -> AirbyteConnectionStatus:
     source = SourceGithub()
@@ -108,10 +110,6 @@ def test_get_org_repositories():
 
     source = SourceGithub()
 
-    with pytest.raises(Exception):
-        config = {"repository": ""}
-        source._get_org_repositories(config, authenticator=None)
-
     responses.add(
         "GET",
         "https://api.github.com/repos/airbytehq/integration-test",
@@ -143,33 +141,68 @@ def test_organization_or_repo_available():
     assert exc_info.value.args[0] == "No streams available. Please check permissions"
 
 
-@pytest.mark.parametrize(
-    ("repos_config", "expected"),
-    [
-        (("airbytehq/airbyte/", "airbytehq/", "airbytehq", "airbyte hq", "airbytehq/*/", "adc/ff*f", "akj*/jahsd"), False),
-        (("airbytehq/airbyte", ), True),
-        (("airbytehq/airbyte-test", "airbytehq/airbyte_test", "airbytehq/airbyte-test/another-repo"), True),
-        (("air232bytehq/air32byte", "airbyte_hq/another-repo", "airbytehq/*", "airbytehq/airbyte"), True),
-        (("airbyte_hq/another.repo", "airbytehq/*", "airbytehq/airbyte"), True),
-    ],
-)
-def test_config_validation(repos_config, expected):
-    actual = SourceGithub._is_repositories_config_valid(repos_config)
-    assert actual == expected
-
-
-@pytest.mark.parametrize(
-    ("config", "expected"),
-    [
-        ({"access_token": "test_token", "repository": "airbytehq/airbyte-test"}, {"airbytehq/airbyte-test", }),
-        ({"access_token": "test_token", "repository": "airbytehq/airbyte-test "}, {"airbytehq/airbyte-test", }),
-        ({"access_token": "test_token", "repository": "airbytehq/airbyte-test/"}, {"airbytehq/airbyte-test", }),
-        ({"access_token": "test_token", "repository": "airbytehq/airbyte-test/  airbytehq/airbyte"}, {"airbytehq/airbyte", "airbytehq/airbyte-test"}),
-    ],
-)
-def tests_get_and_prepare_repositories_config(config, expected):
-    actual = SourceGithub._get_and_prepare_repositories_config(config)
-    assert actual == expected
+def tests_get_and_prepare_repositories_config():
+    config = {"repository": "airbytehq/airbyte airbytehq/airbyte.test  airbytehq/integration-test"}
+    assert SourceGithub._get_and_prepare_repositories_config(config) == {"airbytehq/airbyte", "airbytehq/airbyte.test", "airbytehq/integration-test"}
+
+
+def test_check_config_repository():
+    source = SourceGithub()
+    source.check = MagicMock(return_value=True)
+    config = {"credentials": {"access_token": "access_token"}, "start_date": "1900-01-01T00:00:00Z"}
+
+    repos_ok = [
+        "airbytehq/airbyte",
+        "airbytehq/airbyte-test",
+        "airbytehq/airbyte_test",
+        "erohmensing/thismonth.rocks",
+        "airbytehq/*",
+        "airbytehq/.",
+        "airbyte_hq/airbyte",
+        "airbytehq/123",
+        "airbytehq/airbytexgit",
+    ]
+
+    repos_fail = [
+        "airbytehq",
+        "airbytehq/",
+        "airbytehq/*/",
+        "airbytehq/airbyte.git",
+        "airbytehq/airbyte/",
+        "airbytehq/air*yte",
+        "airbyte*/airbyte",
+        "airbytehq/airbyte-test/master-branch",
+        "https://github.com/airbytehq/airbyte",
+    ]
+
+    config["repository"] = ""
+    with pytest.raises(AirbyteTracedException):
+        assert command_check(source, config)
+    config["repository"] = " "
+    with pytest.raises(AirbyteTracedException):
+        assert command_check(source, config)
+
+    for repos in repos_ok:
+        config["repository"] = repos
+        assert command_check(source, config)
+
+    for repos in repos_fail:
+        config["repository"] = repos
+        with pytest.raises(AirbyteTracedException):
+            assert command_check(source, config)
+
+    config["repository"] = " ".join(repos_ok)
+    assert command_check(source, config)
+    config["repository"] = "    ".join(repos_ok)
+    assert command_check(source, config)
+    config["repository"] = ",".join(repos_ok)
+    with pytest.raises(AirbyteTracedException):
+        assert command_check(source, config)
+
+    for repos in repos_fail:
+        config["repository"] = " ".join(repos_ok[:len(repos_ok)//2] + [repos] + repos_ok[len(repos_ok)//2:])
+        with pytest.raises(AirbyteTracedException):
+            assert command_check(source, config)
 
 
 def test_streams_no_streams_available_error():

airbyte-integrations/connectors/source-github/unit_tests/utils.py

@@ -3,10 +3,14 @@
 #
 
 from typing import Any, MutableMapping
+from unittest import mock
 
 import responses
 from airbyte_cdk.models import SyncMode
+from airbyte_cdk.models.airbyte_protocol import ConnectorSpecification
+from airbyte_cdk.sources import Source
 from airbyte_cdk.sources.streams import Stream
+from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit, split_config
 
 
 def read_incremental(stream_instance: Stream, stream_state: MutableMapping[str, Any]):
@@ -63,3 +67,12 @@ def register(cls, data):
             for n, column in enumerate(project.get("columns", []), start=1):
                 column_id = int(str(project_id) + str(n))
                 responses.upsert("GET", cls.cards_url.format(column_id=column_id), json=cls.get_json_cards(column, column_id))
+
+
+def command_check(source: Source, config):
+    logger = mock.MagicMock()
+    connector_config, _ = split_config(config)
+    if source.check_config_against_spec:
+        source_spec: ConnectorSpecification = source.spec(logger)
+        check_config_against_spec_or_exit(connector_config, source_spec)
+    return source.check(logger, config)

docs/integrations/sources/github.md

@@ -164,6 +164,7 @@ The GitHub connector should not run into GitHub API limitations under normal usa
 
 | Version | Date       | Pull Request                                                                                                      | Subject                                                                                                                                                             |
 |:--------|:-----------|:------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 1.0.0   | 2023-05-19 | [25778](https://github.com/airbytehq/airbyte/pull/25778)                                                          | Improve repo(s) name validation on UI                                                                                                                               |
 | 0.5.0   | 2023-05-16 | [25793](https://github.com/airbytehq/airbyte/pull/25793)                                                          | Implement client-side throttling of requests                                                                                                                        |
 | 0.4.11  | 2023-05-12 | [26025](https://github.com/airbytehq/airbyte/pull/26025)                                                          | Added more transparent depiction of the personal access token expired                                                                                               |
 | 0.4.10  | 2023-05-15 | [26075](https://github.com/airbytehq/airbyte/pull/26075)                                                          | Add more specific error message description for no repos case.                                                                                                      |