internal poetry packages: declare poe tasks and airbyte-ci sections i…

…n pyproject.toml

airbyte-ci/connectors/base_images/pyproject.toml

@@ -32,3 +32,10 @@ generate-docs =  "base_images.commands:generate_docs"
 generate-release =  "base_images.commands:generate_release"
 publish =  "base_images.commands:publish_existing_version"
 
+[tool.poe.tasks]
+test = "pytest tests"
+
+[tool.airbyte_ci]
+extra_poetry_groups = ["dev"]
+poe_tasks = ["test"]
+mount_docker_socket = true

airbyte-ci/connectors/ci_credentials/pyproject.toml

@@ -27,3 +27,10 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry.scripts]
 ci_credentials = "ci_credentials.main:ci_credentials"
+
+[tool.poe.tasks]
+test = "pytest tests"
+
+[tool.airbyte_ci]
+extra_poetry_groups = ["dev"]
+poe_tasks = ["test"]

airbyte-ci/connectors/common_utils/pyproject.toml

@@ -22,3 +22,11 @@ pytest = "^7.2.2"
 [build-system]
 requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"
+
+[tool.poe.tasks]
+test = "pytest tests"
+
+[tool.airbyte_ci]
+extra_poetry_groups = ["dev"]
+# Disable poe tasks as tests are not passing ATM
+poe_tasks = []

airbyte-ci/connectors/connector_ops/pyproject.toml

@@ -34,3 +34,10 @@ write-review-requirements-file = "connector_ops.required_reviewer_checks:write_r
 print-mandatory-reviewers = "connector_ops.required_reviewer_checks:print_mandatory_reviewers"
 allowed-hosts-checks = "connector_ops.allowed_hosts_checks:check_allowed_hosts"
 run-qa-checks = "connector_ops.qa_checks:run_qa_checks"
+
+[tool.poe.tasks]
+test = "pytest tests"
+
+[tool.airbyte_ci]
+extra_poetry_groups = ["dev"]
+poe_tasks = ["test"]

airbyte-ci/connectors/metadata_service/lib/pyproject.toml

@@ -18,6 +18,7 @@ pydash = "^6.0.2"
 semver = "^3.0.1"
 
 
+
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.2.2"
 datamodel-code-generator = "^0.17.1"
@@ -32,6 +33,11 @@ generate-models = { shell = "./bin/generate-python-classes.sh" }
 replicate-prod = "gsutil -m rsync -r -d gs://prod-airbyte-cloud-connector-metadata-service gs://$TARGET_BUCKET"
 copy-connector-from-prod = "gsutil  -m rsync -r -d gs://prod-airbyte-cloud-connector-metadata-service/metadata/$CONNECTOR/$VERSION gs://$TARGET_BUCKET/metadata/$CONNECTOR/$VERSION"
 promote-connector-to-latest = "gsutil  -m rsync -r -d gs://$TARGET_BUCKET/metadata/$CONNECTOR/$VERSION gs://$TARGET_BUCKET/metadata/$CONNECTOR/latest"
+test = "pytest tests"
+
+[tool.airbyte_ci]
+extra_poetry_groups = ["dev"]
+poe_tasks = ["test"]
 
 [build-system]
 requires = ["poetry-core"]

airbyte-ci/connectors/metadata_service/orchestrator/pyproject.toml

@@ -44,3 +44,10 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.dagster]
 module_name = "orchestrator"
+
+[tool.poe.tasks]
+test = "pytest tests"
+
+[tool.airbyte_ci]
+extra_poetry_groups = ["dev"]
+poe_tasks = ["test"]

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/models.py

@@ -0,0 +1,36 @@
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+
+import os
+from typing import Dict, Set
+
+from pydantic import BaseModel, Field, validator
+
+
+class AirbyteCiPackageConfiguration(BaseModel):
+    poe_tasks: Set[str] = Field(..., description="List of unique poe tasks to run")
+    required_environment_variables: Set[str] = Field(
+        set(), description="List of unique required environment variables to pass to the container running the poe task"
+    )
+    extra_poetry_groups: Set[str] = Field(set(), description="List of unique extra poetry groups to install")
+    side_car_docker_engine: bool = Field(
+        False, description="Flag indicating the use of a sidecar Docker engine during the poe task executions"
+    )
+    mount_docker_socket: bool = Field(
+        False,
+        description="Flag indicating the mount of the host docker socket to the container running the poe task, useful when the package under test is using dagger",
+    )
+
+    @validator("required_environment_variables")
+    def check_required_environment_variables_are_set(cls, value: Set) -> Set:
+        for required_env_var in value:
+            if required_env_var not in os.environ:
+                raise ValueError(f"Environment variable {required_env_var} is not set.")
+        return value
+
+
+def deserialize_airbyte_ci_config(pyproject_toml: Dict) -> AirbyteCiPackageConfiguration:
+    try:
+        airbyte_ci_config = pyproject_toml["tool"]["airbyte_ci"]
+    except KeyError:
+        raise ValueError("Missing tool.airbyte_ci configuration in pyproject.toml")
+    return AirbyteCiPackageConfiguration.parse_obj(airbyte_ci_config)

airbyte-ci/connectors/pipelines/pyproject.toml

@@ -41,6 +41,8 @@ pytest = "^6.2.5"
 pytest-mock = "^3.10.0"
 mypy = "^1.7.1"
 ruff = "^0.1.9"
+types-toml = "^0.10.8"
+types-requests = "2.28.2"
 
 [tool.poetry.scripts]
 airbyte-ci = "pipelines.cli.airbyte_ci:airbyte_ci"
@@ -49,3 +51,13 @@ airbyte-ci-dev = "pipelines.cli.airbyte_ci:airbyte_ci"
 [tool.poe.tasks.build-release-binary]
 shell = "pyinstaller --additional-hooks-dir=pyinstaller_hooks --collect-all pipelines --collect-all beartype --collect-all dagger --hidden-import strawberry --name $ARTIFACT_NAME --onefile pipelines/cli/airbyte_ci.py"
 args = [{name = "ARTIFACT_NAME", default="airbyte-ci", positional = true}]
+
+[tool.poe.tasks]
+test = "pytest tests"
+type_check = "mypy pipelines --disallow-untyped-defs"
+lint = "ruff check pipelines"
+
+[tool.airbyte_ci]
+extra_poetry_groups = ["dev"]
+poe_tasks = ["type_check", "lint", "test"]
+mount_docker_socket = true

airbyte-integrations/bases/connector-acceptance-test/pyproject.toml

@@ -41,4 +41,10 @@ urllib3 = "<2.0"
 requests = "<2.29.0"
 pytest-xdist = "^3.3.1"
 
-[tool.poetry.dev-dependencies]
+[tool.poe.tasks]
+test = "pytest unit_tests"
+ci = ["test"]
+
+[tool.airbyte_ci]
+poe_tasks = ["test"]
+mount_docker_socket = true

airbyte-lib/pyproject.toml

@@ -241,3 +241,12 @@ ignore_missing_imports = true  # No stubs yet (😢)
 [tool.poetry.scripts]
 generate-docs = "docs:run"
 airbyte-lib-validate-source = "airbyte_lib.validate:run"
+
+[tool.poe.tasks]
+test = "pytest tests"
+
+[tool.airbyte_ci]
+extra_poetry_groups = ["dev"]
+poe_tasks = ["test"]
+required_environment_variables = ["GCP_GSM_CREDENTIALS"]
+side_car_docker_engine = true