connectors-ci: improve global status reporting to GitHub for tests (#…

…26575)

.github/workflows/metadata_validate_manifest_dagger.yml

@@ -11,6 +11,7 @@ jobs:
     runs-on: medium-runner
     env:
       CI_GITHUB_ACCESS_TOKEN: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
+      PRODUCTION: "true"
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v2

tools/ci_connector_ops/ci_connector_ops/pipelines/bases.py

@@ -28,6 +28,15 @@
     from ci_connector_ops.pipelines.contexts import ConnectorContext, PipelineContext
 
 
+class CIContext(str, Enum):
+    """An enum for Ci context values which can be ["manual", "pull_request", "nightly_builds"]."""
+
+    MANUAL = "manual"
+    PULL_REQUEST = "pull_request"
+    NIGHTLY_BUILDS = "nightly_builds"
+    MASTER = "master"
+
+
 class StepStatus(Enum):
     """An Enum to characterize the success, failure or skipping of a Step."""
 

tools/ci_connector_ops/ci_connector_ops/pipelines/commands/airbyte_ci.py

@@ -5,7 +5,7 @@
 """This module is the CLI entrypoint to the airbyte-ci commands."""
 
 import click
-from ci_connector_ops.pipelines.contexts import CIContext
+from ci_connector_ops.pipelines.bases import CIContext
 from ci_connector_ops.pipelines.utils import (
     get_current_epoch_time,
     get_current_git_branch,

tools/ci_connector_ops/ci_connector_ops/pipelines/commands/groups/connectors.py

@@ -6,16 +6,14 @@
 
 import logging
 import os
-import sys
 from pathlib import Path
 from typing import Any, Dict, Tuple
 
 import anyio
 import click
-import dagger
 from ci_connector_ops.pipelines.builds import run_connector_build_pipeline
-from ci_connector_ops.pipelines.contexts import CIContext, ConnectorContext, ContextState, PublishConnectorContext
-from ci_connector_ops.pipelines.github import update_commit_status_check
+from ci_connector_ops.pipelines.contexts import ConnectorContext, ContextState, PublishConnectorContext
+from ci_connector_ops.pipelines.github import update_global_commit_status_check_for_tests
 from ci_connector_ops.pipelines.pipelines.connectors import run_connectors_pipelines
 from ci_connector_ops.pipelines.publish import run_connector_publish_pipeline
 from ci_connector_ops.pipelines.tests import run_connector_test_pipeline
@@ -25,11 +23,6 @@
 from rich.table import Table
 from rich.text import Text
 
-# CONSTANTS
-
-GITHUB_GLOBAL_CONTEXT = "[POC please ignore] Connectors CI"
-GITHUB_GLOBAL_DESCRIPTION = "Running connectors tests"
-
 logging.basicConfig(level=logging.INFO, format="%(name)s: %(message)s", datefmt="[%X]", handlers=[RichHandler(rich_tracebacks=True)])
 
 logger = logging.getLogger(__name__)
@@ -107,15 +100,6 @@ def connectors(
     ctx.obj["modified"] = modified
     ctx.obj["concurrency"] = concurrency
     ctx.obj["execute_timeout"] = execute_timeout
-    update_commit_status_check(
-        ctx.obj["git_revision"],
-        "pending",
-        ctx.obj["gha_workflow_run_url"],
-        GITHUB_GLOBAL_DESCRIPTION,
-        GITHUB_GLOBAL_CONTEXT,
-        should_send=ctx.obj["ci_context"] == CIContext.PULL_REQUEST,
-        logger=logger,
-    )
 
     all_connectors = get_all_released_connectors()
 
@@ -145,9 +129,6 @@ def connectors(
         selected_connectors_and_files = {
             connector: modified_files for connector, modified_files in selected_connectors_and_files.items() if modified_files
         }
-    if not selected_connectors_and_files:
-        click.secho("No connector were selected according to your inputs. Please double check your filters.", fg="yellow")
-        sys.exit(0)
 
     ctx.obj["selected_connectors_and_files"] = selected_connectors_and_files
     ctx.obj["selected_connectors_names"] = [c.technical_name for c in selected_connectors_and_files.keys()]
@@ -164,10 +145,16 @@ def test(
         ctx (click.Context): The click context.
     """
     click.secho(f"Will run the test pipeline for the following connectors: {', '.join(ctx.obj['selected_connectors_names'])}.", fg="green")
+    if ctx.obj["selected_connectors_and_files"]:
+        update_global_commit_status_check_for_tests(ctx.obj, "pending")
+    else:
+        click.secho("No connector were selected for testing.", fg="yellow")
+        update_global_commit_status_check_for_tests(ctx.obj, "success")
+        return True
 
     connectors_tests_contexts = [
         ConnectorContext(
-            pipeline_name="Test",
+            pipeline_name=f"Testing connector {connector.technical_name}",
             connector=connector,
             is_local=ctx.obj["is_local"],
             git_branch=ctx.obj["git_branch"],
@@ -190,27 +177,13 @@ def test(
             ctx.obj["concurrency"],
             ctx.obj["execute_timeout"],
         )
-        update_commit_status_check(
-            ctx.obj["git_revision"],
-            "success",
-            ctx.obj["gha_workflow_run_url"],
-            GITHUB_GLOBAL_DESCRIPTION,
-            GITHUB_GLOBAL_CONTEXT,
-            should_send=ctx.obj.get("ci_context") == CIContext.PULL_REQUEST,
-            logger=logger,
-        )
-    except dagger.DaggerError as e:
+    except Exception as e:
         click.secho(str(e), err=True, fg="red")
-        update_commit_status_check(
-            ctx.obj["git_revision"],
-            "error",
-            ctx.obj["gha_workflow_run_url"],
-            GITHUB_GLOBAL_DESCRIPTION,
-            GITHUB_GLOBAL_CONTEXT,
-            should_send=ctx.obj.get("ci_context") == CIContext.PULL_REQUEST,
-            logger=logger,
-        )
+        update_global_commit_status_check_for_tests(ctx.obj, "failure")
         return False
+    global_success = all(connector_context.state is ContextState.SUCCESSFUL for connector_context in connectors_tests_contexts)
+    update_global_commit_status_check_for_tests(ctx.obj, "success" if global_success else "failure")
+    # If we reach this point, it means that all the connectors have been tested so the pipeline did its job and can exit with success.
     return True
 
 
@@ -220,7 +193,7 @@ def build(ctx: click.Context) -> bool:
     click.secho(f"Will build the following connectors: {', '.join(ctx.obj['selected_connectors_names'])}.", fg="green")
     connectors_contexts = [
         ConnectorContext(
-            pipeline_name="Build",
+            pipeline_name="Build connector {connector.technical_name}",
             connector=connector,
             is_local=ctx.obj["is_local"],
             git_branch=ctx.obj["git_branch"],

tools/ci_connector_ops/ci_connector_ops/pipelines/commands/groups/metadata.py

@@ -5,7 +5,7 @@
 
 import anyio
 import click
-from ci_connector_ops.pipelines.contexts import CIContext
+from ci_connector_ops.pipelines.bases import CIContext
 from ci_connector_ops.pipelines.pipelines.metadata import (
     run_metadata_lib_test_pipeline,
     run_metadata_orchestrator_deploy_pipeline,

tools/ci_connector_ops/ci_connector_ops/pipelines/contexts.py

@@ -16,23 +16,14 @@
 from anyio import Path
 from asyncer import asyncify
 from ci_connector_ops.pipelines.actions import remote_storage, secrets
-from ci_connector_ops.pipelines.bases import ConnectorReport, Report
+from ci_connector_ops.pipelines.bases import CIContext, ConnectorReport, Report
 from ci_connector_ops.pipelines.github import update_commit_status_check
 from ci_connector_ops.pipelines.slack import send_message_to_webhook
 from ci_connector_ops.pipelines.utils import AIRBYTE_REPO_URL, METADATA_FILE_NAME, sanitize_gcs_credentials
 from ci_connector_ops.utils import Connector
 from dagger import Client, Directory, Secret
 
 
-class CIContext(str, Enum):
-    """An enum for Ci context values which can be ["manual", "pull_request", "nightly_builds"]."""
-
-    MANUAL = "manual"
-    PULL_REQUEST = "pull_request"
-    NIGHTLY_BUILDS = "nightly_builds"
-    MASTER = "master"
-
-
 class ContextState(Enum):
     """Enum to characterize the current context state, values are used for external representation on GitHub commit checks."""
 
@@ -313,8 +304,6 @@ def __init__(
             gha_workflow_run_url=gha_workflow_run_url,
             pipeline_start_timestamp=pipeline_start_timestamp,
             ci_context=ci_context,
-            # TODO: remove this once stable and our default pipeline
-            is_ci_optional=True,
             slack_webhook=slack_webhook,
             reporting_slack_channel=reporting_slack_channel,
         )

tools/ci_connector_ops/ci_connector_ops/pipelines/github.py

@@ -9,6 +9,7 @@
 import os
 from typing import TYPE_CHECKING, Optional
 
+from ci_connector_ops.pipelines.bases import CIContext
 from ci_connector_ops.utils import console
 
 if TYPE_CHECKING:
@@ -17,6 +18,8 @@
 from github import Github
 
 AIRBYTE_GITHUB_REPO = "airbytehq/airbyte"
+GITHUB_GLOBAL_CONTEXT_FOR_TESTS = "Connectors CI tests"
+GITHUB_GLOBAL_DESCRIPTION_FOR_TESTS = "Running connectors tests"
 
 
 def safe_log(logger: Optional[Logger], message: str, level: str = "info") -> None:
@@ -63,10 +66,23 @@ def update_commit_status_check(
         state = "success"
         description = f"[WARNING] optional check failed {context}: {description}"
 
+    context = context if bool(os.environ.get("PRODUCTION", False)) is True else f"[please ignore] {context}"
     airbyte_repo.get_commit(sha=sha).create_status(
         state=state,
         target_url=target_url,
         description=description,
         context=context,
     )
     safe_log(logger, f"Created {state} status for commit {sha} on Github in {context} context with desc: {description}.")
+
+
+def update_global_commit_status_check_for_tests(click_context: dict, github_state: str, logger: Logger = None):
+    update_commit_status_check(
+        click_context["git_revision"],
+        github_state,
+        click_context["gha_workflow_run_url"],
+        GITHUB_GLOBAL_DESCRIPTION_FOR_TESTS,
+        GITHUB_GLOBAL_CONTEXT_FOR_TESTS,
+        should_send=click_context.get("ci_context") == CIContext.PULL_REQUEST,
+        logger=logger,
+    )

tools/ci_connector_ops/ci_connector_ops/pipelines/pipelines/connectors.py

@@ -28,6 +28,6 @@ async def run_connectors_pipelines(
     async with dagger.Connection(Config(log_output=sys.stderr, execute_timeout=execute_timeout)) as dagger_client:
         async with anyio.create_task_group() as tg:
             for context in contexts:
-                context.dagger_client = dagger_client.pipeline(f"{context.connector.technical_name} - {pipeline_name}")
+                context.dagger_client = dagger_client.pipeline(pipeline_name)
                 tg.start_soon(connector_pipeline, context, semaphore, *args)
     return contexts

tools/ci_connector_ops/ci_connector_ops/pipelines/tests/common.py

@@ -13,8 +13,7 @@
 import semver
 import yaml
 from ci_connector_ops.pipelines.actions import environments
-from ci_connector_ops.pipelines.bases import PytestStep, Step, StepResult, StepStatus
-from ci_connector_ops.pipelines.contexts import CIContext
+from ci_connector_ops.pipelines.bases import CIContext, PytestStep, Step, StepResult, StepStatus
 from ci_connector_ops.pipelines.utils import METADATA_FILE_NAME
 from ci_connector_ops.utils import Connector
 from dagger import File