airbyte-lib: Add testing to connectors (airbytehq#34044)

airbyte-ci/connectors/pipelines/README.md

@@ -261,13 +261,15 @@ flowchart TD
         build[Build connector docker image]
         unit[Run unit tests]
         integration[Run integration tests]
+        airbyte_lib_validation[Run airbyte-lib validation tests]
         cat[Run connector acceptance tests]
         secret[Load connector configuration]
 
         unit-->secret
         unit-->build
         secret-->integration
         secret-->cat
+        secret-->airbyte_lib_validation
         build-->integration
         build-->cat
     end
@@ -610,6 +612,7 @@ E.G.: running `pytest` on a specific test folder:
 
 | Version | PR                                                         | Description                                                                                                                |
 | ------- | ---------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- |
+| 3.10.2  | [#34044](https://github.com/airbytehq/airbyte/pull/34044)  | Add pypi validation testing.                                                                                               |
 | 3.10.1  | [#34756](https://github.com/airbytehq/airbyte/pull/34756)  | Enable connectors tests in draft PRs.                                                                                      |
 | 3.10.0  | [#34606](https://github.com/airbytehq/airbyte/pull/34606)  | Allow configuration of separate check URL to check whether package exists already.                                         |
 | 3.9.0   | [#34606](https://github.com/airbytehq/airbyte/pull/34606)  | Allow configuration of python registry URL via environment variable.                                                       |

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/consts.py

@@ -14,6 +14,7 @@ class CONNECTOR_TEST_STEP_ID(str, Enum):
     BUILD = "build"
     CHECK_BASE_IMAGE = "check_base_image"
     INTEGRATION = "integration"
+    AIRBYTE_LIB_VALIDATION = "airbyte_lib_validation"
     METADATA_VALIDATION = "metadata_validation"
     QA_CHECKS = "qa_checks"
     UNIT = "unit"

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py

@@ -7,15 +7,18 @@
 from abc import ABC, abstractmethod
 from typing import List, Sequence, Tuple
 
+import dpath.util
 import pipelines.dagger.actions.python.common
 import pipelines.dagger.actions.system.docker
 from dagger import Container, File
+from pipelines import hacks
 from pipelines.airbyte_ci.connectors.build_image.steps.python_connectors import BuildConnectorImages
 from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID
 from pipelines.airbyte_ci.connectors.context import ConnectorContext
 from pipelines.airbyte_ci.connectors.test.steps.common import AcceptanceTests, CheckBaseImageIsUsed
 from pipelines.consts import LOCAL_BUILD_PLATFORM
 from pipelines.dagger.actions import secrets
+from pipelines.dagger.actions.python.poetry import with_poetry
 from pipelines.helpers.execution.run_steps import STEP_TREE, StepToRun
 from pipelines.models.steps import STEP_PARAMS, Step, StepResult
 
@@ -189,6 +192,49 @@ def default_params(self) -> STEP_PARAMS:
         return super().default_params | coverage_options
 
 
+class AirbyteLibValidation(Step):
+    """A step to validate the connector will work with airbyte-lib, using the airbyte-lib validation helper."""
+
+    title = "AirbyteLib validation tests"
+
+    context: ConnectorContext
+
+    async def _run(self, connector_under_test: Container) -> StepResult:
+        """Run all pytest tests declared in the test directory of the connector code.
+        Args:
+            connector_under_test (Container): The connector under test container.
+        Returns:
+            StepResult: Failure or success of the unit tests with stdout and stdout.
+        """
+        if dpath.util.get(self.context.connector.metadata, "remoteRegistries/pypi/enabled", default=False) is False:
+            return self.skip("Connector is not published on pypi, skipping airbyte-lib validation.")
+
+        test_environment = await self.install_testing_environment(with_poetry(self.context))
+        test_execution = test_environment.with_(
+            hacks.never_fail_exec(["airbyte-lib-validate-source", "--connector-dir", ".", "--validate-install-only"])
+        )
+
+        return await self.get_step_result(test_execution)
+
+    async def install_testing_environment(
+        self,
+        built_connector_container: Container,
+    ) -> Container:
+        """Add airbyte-lib and secrets to the test environment."""
+        context: ConnectorContext = self.context
+
+        container_with_test_deps = await pipelines.dagger.actions.python.common.with_python_package(
+            self.context, built_connector_container.with_entrypoint([]), str(context.connector.code_directory)
+        )
+        return container_with_test_deps.with_exec(
+            [
+                "pip",
+                "install",
+                "airbyte-lib",
+            ]
+        )
+
+
 class IntegrationTests(PytestStep):
     """A step to run the connector integration tests with Pytest."""
 
@@ -218,6 +264,12 @@ def get_test_steps(context: ConnectorContext) -> STEP_TREE:
                 args=lambda results: {"connector_under_test": results[CONNECTOR_TEST_STEP_ID.BUILD].output_artifact[LOCAL_BUILD_PLATFORM]},
                 depends_on=[CONNECTOR_TEST_STEP_ID.BUILD],
             ),
+            StepToRun(
+                id=CONNECTOR_TEST_STEP_ID.AIRBYTE_LIB_VALIDATION,
+                step=AirbyteLibValidation(context),
+                args=lambda results: {"connector_under_test": results[CONNECTOR_TEST_STEP_ID.BUILD].output_artifact[LOCAL_BUILD_PLATFORM]},
+                depends_on=[CONNECTOR_TEST_STEP_ID.BUILD],
+            ),
             StepToRun(
                 id=CONNECTOR_TEST_STEP_ID.ACCEPTANCE,
                 step=AcceptanceTests(context, context.concurrent_cat),

airbyte-ci/connectors/pipelines/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "3.10.1"
+version = "3.10.2"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <contact@airbyte.io>"]
 
@@ -30,6 +30,7 @@ certifi = "^2023.11.17"
 tomli = "^2.0.1"
 tomli-w = "^1.0.0"
 types-requests = "2.28.2"
+dpath = "^2.1.6"
 
 [tool.poetry.group.dev.dependencies]
 freezegun = "^1.2.2"

airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py

@@ -2,12 +2,14 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
+from unittest.mock import patch
+
 import pytest
 from connector_ops.utils import Connector, ConnectorLanguage
 from pipelines.airbyte_ci.connectors.build_image.steps.python_connectors import BuildConnectorImages
 from pipelines.airbyte_ci.connectors.context import ConnectorContext
-from pipelines.airbyte_ci.connectors.test.steps.python_connectors import UnitTests
-from pipelines.models.steps import StepResult
+from pipelines.airbyte_ci.connectors.test.steps.python_connectors import AirbyteLibValidation, UnitTests
+from pipelines.models.steps import StepResult, StepStatus
 
 pytestmark = [
     pytest.mark.anyio,
@@ -105,3 +107,72 @@ def test_params(self, context_for_certified_connector_with_setup):
             f"--cov={context_for_certified_connector_with_setup.connector.technical_name.replace('-', '_')}",
             f"--cov-fail-under={step.MINIMUM_COVERAGE_FOR_CERTIFIED_CONNECTORS}",
         ]
+
+
+class TestAirbyteLibValidationTests:
+    @pytest.fixture
+    def compatible_connector(self):
+        return Connector("source-faker")
+
+    @pytest.fixture
+    def incompatible_connector(self):
+        return Connector("source-postgres")
+
+    @pytest.fixture
+    def context_for_valid_connector(self, compatible_connector, dagger_client, current_platform):
+        context = ConnectorContext(
+            pipeline_name="test airbyte-lib validation",
+            connector=compatible_connector,
+            git_branch="test",
+            git_revision="test",
+            report_output_prefix="test",
+            is_local=True,
+            use_remote_secrets=True,
+            targeted_platforms=[current_platform],
+        )
+        context.dagger_client = dagger_client
+        return context
+
+    @pytest.fixture
+    def context_for_invalid_connector(self, incompatible_connector, dagger_client, current_platform):
+        context = ConnectorContext(
+            pipeline_name="test airbyte-lib validation",
+            connector=incompatible_connector,
+            git_branch="test",
+            git_revision="test",
+            report_output_prefix="test",
+            is_local=True,
+            use_remote_secrets=True,
+            targeted_platforms=[current_platform],
+        )
+        context.dagger_client = dagger_client
+        return context
+
+    async def test__run_validation_success(self, mocker, context_for_valid_connector: ConnectorContext):
+        result = await AirbyteLibValidation(context_for_valid_connector)._run(mocker.MagicMock())
+        assert isinstance(result, StepResult)
+        assert result.status == StepStatus.SUCCESS
+        assert "Creating source and validating spec is returned successfully..." in result.stdout
+
+    async def test__run_validation_skip_unpublished_connector(
+        self,
+        mocker,
+        context_for_invalid_connector: ConnectorContext,
+    ):
+        result = await AirbyteLibValidation(context_for_invalid_connector)._run(mocker.MagicMock())
+        assert isinstance(result, StepResult)
+        assert result.status == StepStatus.SKIPPED
+
+    async def test__run_validation_fail(
+        self,
+        mocker,
+        context_for_invalid_connector: ConnectorContext,
+    ):
+        metadata = context_for_invalid_connector.connector.metadata
+        metadata["remoteRegistries"] = {"pypi": {"enabled": True, "packageName": "airbyte-source-postgres"}}
+        metadata_mock = mocker.PropertyMock(return_value=metadata)
+        with patch.object(Connector, "metadata", metadata_mock):
+            result = await AirbyteLibValidation(context_for_invalid_connector)._run(mocker.MagicMock())
+            assert isinstance(result, StepResult)
+            assert result.status == StepStatus.FAILURE
+            assert "does not appear to be a Python project" in result.stderr