airbyte-ci: Test pypi published properly (#34689)

.github/actions/run-airbyte-ci/action.yml

@@ -79,10 +79,6 @@ inputs:
   python_registry_token:
     description: "Python registry API token to publish python package"
     required: false
-  python_registry_url:
-    description: "Python registry URL to publish python package"
-    default: "https://upload.pypi.org/legacy/"
-    required: false
 
 runs:
   using: "composite"
@@ -140,6 +136,7 @@ runs:
         PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }}
         PYTHON_REGISTRY_TOKEN: ${{ inputs.python_registry_token }}
         PYTHON_REGISTRY_URL: ${{ inputs.python_registry_url }}
+        PYTHON_REGISTRY_CHECK_URL: ${{ inputs.python_registry_check_url }}
         S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ inputs.s3_build_cache_access_key_id }}
         S3_BUILD_CACHE_SECRET_KEY: ${{ inputs.s3_build_cache_secret_key }}
         SENTRY_DSN: ${{ inputs.sentry_dsn }}

airbyte-ci/connectors/pipelines/README.md

@@ -397,6 +397,7 @@ Publish all connectors modified in the head commit: `airbyte-ci connectors --mod
 | `--ci-requirements`                  | False    |                                 |                                    | Output the CI requirements as a JSON payload. It is used to determine the CI runner to use.                                                                                               |
 | `--python-registry-token`            | False    |                                 | `PYTHON_REGISTRY_TOKEN`            | The API token to authenticate with the registry. For pypi, the `pypi-` prefix needs to be specified      |
 | `--python-registry-url`              | False    | https://upload.pypi.org/legacy/ | `PYTHON_REGISTRY_URL`              | The python registry to publish to. Defaults to main pypi                                                 |
+| `--python-registry-check-url`        | False    | https://pypi.org/pypi           | `PYTHON_REGISTRY_CHECK_URL`        | The python registry url to check whether a package is published already                                                 |
 
 
 I've added an empty "Default" column, and you can fill in the default values as needed.
@@ -421,7 +422,7 @@ flowchart TD
 
 If `remoteRegistries.pypi.enabled` in the connector metadata is set to `true`, the connector will be published to the python registry.
 To do so, the `--python-registry-token` and `--python-registry-url` options are used to authenticate with the registry and publish the connector.
-If the current version of the connector is already published to the registry, the publish will be skipped.
+If the current version of the connector is already published to the registry, the publish will be skipped (the `--python-registry-check-url` is used for the check).
 
 On a pre-release, the connector will be published as a `.dev<N>` version.
 
@@ -609,6 +610,7 @@ E.G.: running `pytest` on a specific test folder:
 
 | Version | PR                                                         | Description                                                                                                                |
 | ------- | ---------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- |
+| 3.10.0  | [#34606](https://github.com/airbytehq/airbyte/pull/34606)  | Allow configuration of separate check URL to check whether package exists already.                                         |
 | 3.9.0   | [#34606](https://github.com/airbytehq/airbyte/pull/34606)  | Allow configuration of python registry URL via environment variable.                                                       |
 | 3.8.1   | [#34607](https://github.com/airbytehq/airbyte/pull/34607)  | Improve gradle dependency cache volume protection.                                                                         |
 | 3.8.0   | [#34316](https://github.com/airbytehq/airbyte/pull/34316)  | Expose Dagger engine image name in `--ci-requirements` and add `--ci-requirements` to the `airbyte-ci` root command group. |

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py

@@ -11,7 +11,7 @@
 from pipelines.cli.click_decorators import click_ci_requirements_option
 from pipelines.cli.confirm_prompt import confirm
 from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand
-from pipelines.consts import DEFAULT_PYTHON_PACKAGE_REGISTRY_URL, ContextState
+from pipelines.consts import DEFAULT_PYTHON_PACKAGE_REGISTRY_CHECK_URL, DEFAULT_PYTHON_PACKAGE_REGISTRY_URL, ContextState
 from pipelines.helpers.utils import fail_if_missing_docker_hub_creds
 
 
@@ -67,11 +67,18 @@
 )
 @click.option(
     "--python-registry-url",
-    help="Which python registry registry to publish to. If not set, the default pypi is used. For test pypi, use https://test.pypi.org/legacy/",
+    help="Which python registry url to publish to. If not set, the default pypi is used. For test pypi, use https://test.pypi.org/legacy/",
     type=click.STRING,
     default=DEFAULT_PYTHON_PACKAGE_REGISTRY_URL,
     envvar="PYTHON_REGISTRY_URL",
 )
+@click.option(
+    "--python-registry-check-url",
+    help="Which url to check whether a certain version is published already. If not set, the default pypi is used. For test pypi, use https://test.pypi.org/pypi/",
+    type=click.STRING,
+    default=DEFAULT_PYTHON_PACKAGE_REGISTRY_CHECK_URL,
+    envvar="PYTHON_REGISTRY_CHECK_URL",
+)
 @click.pass_context
 async def publish(
     ctx: click.Context,
@@ -84,6 +91,7 @@ async def publish(
     slack_channel: str,
     python_registry_token: str,
     python_registry_url: str,
+    python_registry_check_url: str,
 ) -> bool:
     ctx.obj["spec_cache_gcs_credentials"] = spec_cache_gcs_credentials
     ctx.obj["spec_cache_bucket_name"] = spec_cache_bucket_name
@@ -126,6 +134,7 @@ async def publish(
                 use_local_cdk=ctx.obj.get("use_local_cdk"),
                 python_registry_token=python_registry_token,
                 python_registry_url=python_registry_url,
+                python_registry_check_url=python_registry_check_url,
             )
             for connector in ctx.obj["selected_connectors_with_modified_files"]
         ]

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py

@@ -37,6 +37,8 @@ def __init__(
         is_local: bool,
         git_branch: str,
         git_revision: str,
+        python_registry_url: str,
+        python_registry_check_url: str,
         gha_workflow_run_url: Optional[str] = None,
         dagger_logs_url: Optional[str] = None,
         pipeline_start_timestamp: Optional[int] = None,
@@ -47,7 +49,6 @@ def __init__(
         s3_build_cache_secret_key: Optional[str] = None,
         use_local_cdk: bool = False,
         python_registry_token: Optional[str] = None,
-        python_registry_url: Optional[str] = None,
     ) -> None:
         self.pre_release = pre_release
         self.spec_cache_bucket_name = spec_cache_bucket_name
@@ -56,6 +57,7 @@ def __init__(
         self.metadata_service_gcs_credentials = sanitize_gcs_credentials(metadata_service_gcs_credentials)
         self.python_registry_token = python_registry_token
         self.python_registry_url = python_registry_url
+        self.python_registry_check_url = python_registry_check_url
         pipeline_name = f"Publish {connector.technical_name}"
         pipeline_name = pipeline_name + " (pre-release)" if pre_release else pipeline_name
 

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py

@@ -60,7 +60,7 @@ class CheckPythonRegistryPackageDoesNotExist(Step):
 
     async def _run(self) -> StepResult:
         is_published = is_package_published(
-            self.context.package_metadata.name, self.context.package_metadata.version, self.context.registry
+            self.context.package_metadata.name, self.context.package_metadata.version, self.context.registry_check_url
         )
         if is_published:
             return StepResult(

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py

@@ -14,7 +14,7 @@
 from pipelines.airbyte_ci.steps.python_registry import PublishToPythonRegistry
 from pipelines.cli.confirm_prompt import confirm
 from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand
-from pipelines.consts import DEFAULT_PYTHON_PACKAGE_REGISTRY_URL
+from pipelines.consts import DEFAULT_PYTHON_PACKAGE_REGISTRY_CHECK_URL, DEFAULT_PYTHON_PACKAGE_REGISTRY_URL
 from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context
 from pipelines.models.contexts.python_registry_publish import PythonRegistryPublishContext
 from pipelines.models.steps import StepStatus
@@ -87,6 +87,7 @@ async def publish(
         ci_gcs_credentials=ctx.obj["ci_gcs_credentials"],
         python_registry_token=python_registry_token,
         registry=python_registry_url,
+        registry_check_url=DEFAULT_PYTHON_PACKAGE_REGISTRY_CHECK_URL,
         package_path=ctx.obj["package_path"],
         package_name=publish_name,
         version=publish_version,

airbyte-ci/connectors/pipelines/pipelines/consts.py

@@ -61,6 +61,7 @@
 STORAGE_DRIVER = "fuse-overlayfs"
 SETUP_PY_FILE_PATH = "setup.py"
 DEFAULT_PYTHON_PACKAGE_REGISTRY_URL = "https://upload.pypi.org/legacy/"
+DEFAULT_PYTHON_PACKAGE_REGISTRY_CHECK_URL = "https://pypi.org/pypi"
 
 
 class CIContext(str, Enum):

airbyte-ci/connectors/pipelines/pipelines/helpers/pip.py

@@ -1,27 +1,21 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 
 from typing import Optional
-from urllib.parse import urlparse
 
 import requests  # type: ignore
 
 
 def is_package_published(package_name: Optional[str], version: Optional[str], registry_url: str) -> bool:
     """
-    Check if a package with a specific version is published on PyPI or Test PyPI.
-
-    :param package_name: The name of the package to check.
-    :param version: The version of the package.
-    :param test_pypi: Set to True to check on Test PyPI, False for regular PyPI.
-    :return: True if the package is found with the specified version, False otherwise.
+    Check if a package with a specific version is published on a python registry.
     """
     if not package_name or not version:
         return False
 
-    parsed_registry_url = urlparse(registry_url)
-    base_url = f"{parsed_registry_url.scheme}://{parsed_registry_url.netloc}"
-
-    url = f"{base_url}/{package_name}/{version}/json"
+    url = f"{registry_url}/{package_name}/{version}/json"
 
-    response = requests.get(url)
-    return response.status_code == 200
+    try:
+        response = requests.get(url)
+        return response.status_code == 200
+    except requests.exceptions.ConnectionError:
+        return False

airbyte-ci/connectors/pipelines/pipelines/models/contexts/python_registry_publish.py

@@ -21,6 +21,7 @@ class PythonRegistryPublishContext(PipelineContext):
     def __init__(
         self,
         python_registry_token: str,
+        registry_check_url: str,
         package_path: str,
         report_output_prefix: str,
         is_local: bool,
@@ -38,6 +39,7 @@ def __init__(
     ) -> None:
         self.python_registry_token = python_registry_token
         self.registry = registry
+        self.registry_check_url = registry_check_url
         self.package_path = package_path
         self.package_metadata = PythonPackageMetadata(package_name, version)
 
@@ -87,6 +89,7 @@ async def from_publish_connector_context(
         pypi_context = cls(
             python_registry_token=str(connector_context.python_registry_token),
             registry=str(connector_context.python_registry_url),
+            registry_check_url=str(connector_context.python_registry_check_url),
             package_path=str(connector_context.connector.code_directory),
             package_name=current_metadata["remoteRegistries"]["pypi"]["packageName"],
             version=version,

airbyte-ci/connectors/pipelines/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "3.9.0"
+version = "3.10.0"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <contact@airbyte.io>"]
 

airbyte-ci/connectors/pipelines/tests/test_helpers/test_pip.py

@@ -0,0 +1,23 @@
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+
+import pytest
+from pipelines.helpers.pip import is_package_published
+
+
+@pytest.mark.parametrize(
+    "package_name, version, registry_url, expected",
+    [
+        pytest.param(None, None, "https://pypi.org/pypi", False, id="package_name and version are None"),
+        pytest.param(None, "0.2.0", "https://pypi.org/pypi", False, id="package_name is None"),
+        pytest.param("airbyte-source-pokeapi", None, "https://pypi.org/pypi", False, id="version is None"),
+        pytest.param("airbyte-source-pokeapi", "0.2.0", "https://pypi.org/pypi", True, id="published on pypi"),
+        pytest.param("airbyte-source-pokeapi", "0.1.0", "https://pypi.org/pypi", False, id="version not published on pypi"),
+        pytest.param("airbyte-source-nonexisting", "0.1.0", "https://pypi.org/pypi", False, id="package not published on pypi"),
+        pytest.param("airbyte-source-pokeapi", "0.2.1", "https://test.pypi.org/pypi", True, id="published on test.pypi"),
+        pytest.param("airbyte-source-pokeapi", "0.1.0", "https://test.pypi.org/pypi", False, id="version not published on test.pypi"),
+        pytest.param("airbyte-source-nonexisting", "0.1.0", "https://test.pypi.org/pypi", False, id="package not published on test.pypi"),
+        pytest.param("airbyte-source-pokeapi", "0.2.0", "https://some-non-existing-host.com", False, id="host does not exist"),
+    ],
+)
+def test_is_package_published(package_name, version, registry_url, expected):
+    assert is_package_published(package_name, version, registry_url) == expected

airbyte-ci/connectors/pipelines/tests/test_poetry/test_poetry_publish.py

@@ -24,6 +24,7 @@ def context(dagger_client: Client):
         version="0.2.0",
         python_registry_token="test",
         package_name="test",
+        registry_check_url="http://local_registry:8080/",
         registry="http://local_registry:8080/",
         is_local=True,
         git_branch="test",