-
Notifications
You must be signed in to change notification settings - Fork 3.9k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Airbyte-ci: Add skippable connector test steps #32188
Changes from 38 commits
f1732af
866bc56
5a1c1cc
27267a6
3c0b584
bab78ab
2de8d43
f90f002
dd4d275
ab950d3
258b66d
3a518ca
be10290
3505170
e59dc3c
3e54bd4
ef6f26a
5a61d50
bba7057
0abd1eb
45a7e23
fbcf237
1821d91
5361936
7b61277
1a6e3d5
7f293f3
31d98c9
8a5bbb9
5ac61e3
f6ed2a5
8f57adb
fa233f9
cddd975
4ffa61d
13349dd
b99e39c
ffae7e2
485d0c4
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved. | ||
|
||
from enum import Enum | ||
|
||
|
||
class CONNECTOR_TEST_STEP_ID(str, Enum): | ||
""" | ||
An enum for the different step ids of the connector test pipeline. | ||
""" | ||
|
||
ACCEPTANCE = "acceptance" | ||
BUILD_NORMALIZATION = "build_normalization" | ||
BUILD_TAR = "build_tar" | ||
BUILD = "build" | ||
CHECK_BASE_IMAGE = "check_base_image" | ||
INTEGRATION = "integration" | ||
METADATA_VALIDATION = "metadata_validation" | ||
QA_CHECKS = "qa_checks" | ||
UNIT = "unit" | ||
VERSION_FOLLOW_CHECK = "version_follow_check" | ||
VERSION_INC_CHECK = "version_inc_check" | ||
TEST_ORCHESTRATOR = "test_orchestrator" | ||
DEPLOY_ORCHESTRATOR = "deploy_orchestrator" | ||
|
||
def __str__(self) -> str: | ||
return self.value |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -3,15 +3,18 @@ | |
# | ||
|
||
import sys | ||
from typing import List | ||
|
||
import asyncclick as click | ||
from pipelines import main_logger | ||
from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID | ||
from pipelines.airbyte_ci.connectors.context import ConnectorContext | ||
from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines | ||
from pipelines.airbyte_ci.connectors.test.pipeline import run_connector_test_pipeline | ||
from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand | ||
from pipelines.consts import LOCAL_BUILD_PLATFORM, ContextState | ||
from pipelines.helpers.github import update_global_commit_status_check_for_tests | ||
from pipelines.helpers.run_steps import RunStepOptions | ||
from pipelines.helpers.utils import fail_if_missing_docker_hub_creds | ||
|
||
|
||
|
@@ -30,27 +33,27 @@ | |
type=bool, | ||
is_flag=True, | ||
) | ||
@click.option( | ||
"--fast-tests-only", | ||
help="When enabled, slow tests are skipped.", | ||
default=False, | ||
type=bool, | ||
is_flag=True, | ||
) | ||
@click.option( | ||
"--concurrent-cat", | ||
help="When enabled, the CAT tests will run concurrently. Be careful about rate limits", | ||
default=False, | ||
type=bool, | ||
is_flag=True, | ||
) | ||
@click.option( | ||
"--skip-step", | ||
"-x", | ||
multiple=True, | ||
type=click.Choice([step_id.value for step_id in CONNECTOR_TEST_STEP_ID]), | ||
help="Skip a step by name. Can be used multiple times to skip multiple steps.", | ||
) | ||
@click.pass_context | ||
async def test( | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Shall we add a There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Once I make this change you suggested Then --help should allow them to discover the step names |
||
ctx: click.Context, | ||
code_tests_only: bool, | ||
fail_fast: bool, | ||
fast_tests_only: bool, | ||
concurrent_cat: bool, | ||
skip_step: List[str], | ||
) -> bool: | ||
"""Runs a test pipeline for the selected connectors. | ||
|
||
|
@@ -70,6 +73,11 @@ async def test( | |
update_global_commit_status_check_for_tests(ctx.obj, "success") | ||
return True | ||
|
||
run_step_options = RunStepOptions( | ||
fail_fast=fail_fast, | ||
skip_steps=[CONNECTOR_TEST_STEP_ID(step_id) for step_id in skip_step], | ||
) | ||
|
||
connectors_tests_contexts = [ | ||
ConnectorContext( | ||
pipeline_name=f"Testing connector {connector.technical_name}", | ||
|
@@ -86,19 +94,19 @@ async def test( | |
ci_context=ctx.obj.get("ci_context"), | ||
pull_request=ctx.obj.get("pull_request"), | ||
ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], | ||
fail_fast=fail_fast, | ||
fast_tests_only=fast_tests_only, | ||
code_tests_only=code_tests_only, | ||
use_local_cdk=ctx.obj.get("use_local_cdk"), | ||
s3_build_cache_access_key_id=ctx.obj.get("s3_build_cache_access_key_id"), | ||
s3_build_cache_secret_key=ctx.obj.get("s3_build_cache_secret_key"), | ||
docker_hub_username=ctx.obj.get("docker_hub_username"), | ||
docker_hub_password=ctx.obj.get("docker_hub_password"), | ||
concurrent_cat=concurrent_cat, | ||
run_step_options=run_step_options, | ||
targeted_platforms=[LOCAL_BUILD_PLATFORM], | ||
) | ||
for connector in ctx.obj["selected_connectors_with_modified_files"] | ||
] | ||
|
||
try: | ||
await run_connectors_pipelines( | ||
[connector_context for connector_context in connectors_tests_contexts], | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -3,101 +3,68 @@ | |
# | ||
"""This module groups factory like functions to dispatch tests steps according to the connector under test language.""" | ||
|
||
import itertools | ||
from typing import List | ||
|
||
import anyio | ||
import asyncer | ||
from connector_ops.utils import METADATA_FILE_NAME, ConnectorLanguage | ||
from connector_ops.utils import ConnectorLanguage | ||
from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID | ||
from pipelines.airbyte_ci.connectors.context import ConnectorContext | ||
from pipelines.airbyte_ci.connectors.reports import ConnectorReport | ||
from pipelines.airbyte_ci.connectors.test.steps import java_connectors, python_connectors | ||
from pipelines.airbyte_ci.connectors.test.steps.common import QaChecks, VersionFollowsSemverCheck, VersionIncrementCheck | ||
from pipelines.airbyte_ci.metadata.pipeline import MetadataValidation | ||
from pipelines.models.steps import StepResult | ||
from pipelines.helpers.run_steps import StepToRun, run_steps | ||
|
||
LANGUAGE_MAPPING = { | ||
"run_all_tests": { | ||
ConnectorLanguage.PYTHON: python_connectors.run_all_tests, | ||
ConnectorLanguage.LOW_CODE: python_connectors.run_all_tests, | ||
ConnectorLanguage.JAVA: java_connectors.run_all_tests, | ||
} | ||
"get_test_steps": { | ||
ConnectorLanguage.PYTHON: python_connectors.get_test_steps, | ||
ConnectorLanguage.LOW_CODE: python_connectors.get_test_steps, | ||
ConnectorLanguage.JAVA: java_connectors.get_test_steps, | ||
}, | ||
} | ||
|
||
|
||
async def run_metadata_validation(context: ConnectorContext) -> List[StepResult]: | ||
"""Run the metadata validation on a connector. | ||
Args: | ||
context (ConnectorContext): The current connector context. | ||
|
||
Returns: | ||
List[StepResult]: The results of the metadata validation steps. | ||
""" | ||
return [await MetadataValidation(context).run()] | ||
|
||
|
||
async def run_version_checks(context: ConnectorContext) -> List[StepResult]: | ||
"""Run the version checks on a connector. | ||
|
||
Args: | ||
context (ConnectorContext): The current connector context. | ||
|
||
Returns: | ||
List[StepResult]: The results of the version checks steps. | ||
""" | ||
return [await VersionFollowsSemverCheck(context).run(), await VersionIncrementCheck(context).run()] | ||
|
||
|
||
async def run_qa_checks(context: ConnectorContext) -> List[StepResult]: | ||
"""Run the QA checks on a connector. | ||
def get_test_steps(context: ConnectorContext) -> List[StepToRun]: | ||
"""Get all the tests steps according to the connector language. | ||
|
||
Args: | ||
context (ConnectorContext): The current connector context. | ||
|
||
Returns: | ||
List[StepResult]: The results of the QA checks steps. | ||
List[StepResult]: The list of tests steps. | ||
""" | ||
return [await QaChecks(context).run()] | ||
|
||
|
||
async def run_all_tests(context: ConnectorContext) -> List[StepResult]: | ||
"""Run all the tests steps according to the connector language. | ||
|
||
Args: | ||
context (ConnectorContext): The current connector context. | ||
|
||
Returns: | ||
List[StepResult]: The results of the tests steps. | ||
""" | ||
if _run_all_tests := LANGUAGE_MAPPING["run_all_tests"].get(context.connector.language): | ||
return await _run_all_tests(context) | ||
if _get_test_steps := LANGUAGE_MAPPING["get_test_steps"].get(context.connector.language): | ||
return _get_test_steps(context) | ||
else: | ||
context.logger.warning(f"No tests defined for connector language {context.connector.language}!") | ||
return [] | ||
|
||
|
||
async def run_connector_test_pipeline(context: ConnectorContext, semaphore: anyio.Semaphore) -> ConnectorReport: | ||
"""Run a test pipeline for a single connector. | ||
async def run_connector_test_pipeline(context: ConnectorContext, semaphore: anyio.Semaphore): | ||
""" | ||
Compute the steps to run for a connector test pipeline. | ||
""" | ||
|
||
A visual DAG can be found on the README.md file of the pipelines modules. | ||
steps_to_run = get_test_steps(context) | ||
|
||
Args: | ||
context (ConnectorContext): The initialized connector context. | ||
if not context.code_tests_only: | ||
alafanechere marked this conversation as resolved.
Show resolved
Hide resolved
|
||
steps_to_run += [ | ||
[ | ||
StepToRun(id=CONNECTOR_TEST_STEP_ID.METADATA_VALIDATION, step=MetadataValidation(context)), | ||
StepToRun(id=CONNECTOR_TEST_STEP_ID.VERSION_FOLLOW_CHECK, step=VersionFollowsSemverCheck(context)), | ||
StepToRun(id=CONNECTOR_TEST_STEP_ID.VERSION_INC_CHECK, step=VersionIncrementCheck(context)), | ||
StepToRun(id=CONNECTOR_TEST_STEP_ID.QA_CHECKS, step=QaChecks(context)), | ||
] | ||
] | ||
|
||
Returns: | ||
ConnectorReport: The test reports holding tests results. | ||
""" | ||
async with semaphore: | ||
async with context: | ||
async with asyncer.create_task_group() as task_group: | ||
tasks = [task_group.soonify(run_all_tests)(context)] | ||
if not context.code_tests_only: | ||
tasks += [ | ||
task_group.soonify(run_metadata_validation)(context), | ||
task_group.soonify(run_version_checks)(context), | ||
task_group.soonify(run_qa_checks)(context), | ||
] | ||
results = list(itertools.chain(*(task.value for task in tasks))) | ||
result_dict = await run_steps( | ||
runnables=steps_to_run, | ||
options=context.run_step_options, | ||
) | ||
Comment on lines
+62
to
+65
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 💎 |
||
|
||
results = result_dict.values() | ||
context.report = ConnectorReport(context, steps_results=results, name="TEST RESULTS") | ||
|
||
return context.report |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I think it would make sense to expose this parameter at the
PipelineContext
level for reusability. Wdyt?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We do! Im just passing it through to the pipeline context here