-
Notifications
You must be signed in to change notification settings - Fork 3.8k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Showing
7 changed files
with
336 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,93 @@ | ||
| # Copyright (c) 2024 Airbyte, Inc., all rights reserved. | ||
|
|
||
| name: Connector Ops CI - Bump Connector Versions | ||
|
|
||
| on: | ||
| push: | ||
| branches: | ||
| - master | ||
| paths: | ||
| - "airbyte-integrations/connectors/**/.changelog_entries/*" | ||
|
|
||
| jobs: | ||
| bump_connector_versions: | ||
| name: Publish connectors | ||
| runs-on: connector-publish-large | ||
| steps: | ||
| - name: Checkout Airbyte | ||
| uses: actions/checkout@v3 | ||
| - name: Publish modified connectors [On merge to master] | ||
| id: publish-modified-connectors | ||
| if: github.event_name == 'push' | ||
| uses: ./.github/actions/run-airbyte-ci | ||
| with: | ||
| context: "master" | ||
| dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }} | ||
| docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} | ||
| docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} | ||
| gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} | ||
| gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} | ||
| github_token: ${{ secrets.GITHUB_TOKEN }} | ||
| metadata_service_gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} | ||
| sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} | ||
| slack_webhook_url: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }} | ||
| spec_cache_gcs_credentials: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }} | ||
| s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} | ||
| s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} | ||
| subcommand: "connectors bump_version" | ||
| python_registry_token: ${{ secrets.PYPI_TOKEN }} | ||
|
|
||
| set-instatus-incident-on-failure: | ||
| name: Create Instatus Incident on Failure | ||
| runs-on: ubuntu-latest | ||
| needs: | ||
| - bump_connector_versions | ||
| if: ${{ failure() && github.ref == 'refs/heads/master' }} | ||
| steps: | ||
| - name: Call Instatus Webhook | ||
| uses: joelwmale/webhook-action@master | ||
| with: | ||
| url: ${{ secrets.INSTATUS_CONNECTOR_CI_WEBHOOK_URL }} | ||
| body: '{ "trigger": "down", "status": "HASISSUES" }' | ||
|
|
||
| set-instatus-incident-on-success: | ||
| name: Create Instatus Incident on Success | ||
| runs-on: ubuntu-latest | ||
| needs: | ||
| - bump_connector_versions | ||
| if: ${{ success() && github.ref == 'refs/heads/master' }} | ||
| steps: | ||
| - name: Call Instatus Webhook | ||
| uses: joelwmale/webhook-action@master | ||
| with: | ||
| url: ${{ secrets.INSTATUS_CONNECTOR_CI_WEBHOOK_URL }} | ||
| body: '{ "trigger": "up" }' | ||
|
|
||
| notify-failure-slack-channel: | ||
| name: "Notify Slack Channel on Build Failures" | ||
| runs-on: ubuntu-latest | ||
| needs: | ||
| - bump_connector_versions | ||
| if: ${{ failure() && github.ref == 'refs/heads/master' }} | ||
| steps: | ||
| - name: Checkout Airbyte | ||
| uses: actions/checkout@v3 | ||
| - name: Match GitHub User to Slack User | ||
| id: match-github-to-slack-user | ||
| uses: ./.github/actions/match-github-to-slack-user | ||
| env: | ||
| AIRBYTE_TEAM_BOT_SLACK_TOKEN: ${{ secrets.SLACK_AIRBYTE_TEAM_READ_USERS }} | ||
| GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} | ||
| - name: Publish to OSS Build Failure Slack Channel | ||
| uses: abinoda/slack-action@master | ||
| env: | ||
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} | ||
| with: | ||
| args: >- | ||
| {\"channel\":\"C056HGD1QSW\", \"blocks\":[ | ||
| {\"type\":\"divider\"}, | ||
| {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" Connector Version Bump Failed! :bangbang: \n\n\"}}, | ||
| {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}}, | ||
| {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"<@${{ steps.match-github-to-slack-user.outputs.slack_user_ids }}> \n\"}}, | ||
| {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-shocked: <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|View Action Run> :octavia-shocked: \n\"}}, | ||
| {\"type\":\"divider\"}]} |
24 changes: 24 additions & 0 deletions
24
.github/workflows/create_connector_changelog_entry_files.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| name: Create connector changelog entry files | ||
| run-name: Create connector changelog entry files | ||
| on: | ||
| pull_request: | ||
| types:[opened, edited, synchronize] | ||
|
|
||
| env: | ||
| GITREF: ${{ github.event.inputs.gitref || github.ref }} | ||
|
|
||
|
|
||
| jobs: | ||
| create-connector-changelog-entry-files: | ||
| # IMPORTANT: This name must match the require check name on the branch protection settings | ||
| name: "Delete existing files for the current PR" | ||
| runs-on: tooling-test-small | ||
| steps: | ||
| - name: Checkout Airbyte | ||
| uses: actions/checkout@v3 | ||
| with: | ||
| ref: ${{ env.GITREF }} | ||
|
|
||
| - name: Write PR body to file | ||
|
|
||
| - name: Create Changelog Entry Files |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
187 changes: 187 additions & 0 deletions
187
airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/cdk_java/commands.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,187 @@ | ||
| # | ||
| # Copyright (c) 2023 Airbyte, Inc., all rights reserved. | ||
| # | ||
|
|
||
| import os | ||
| from pathlib import Path | ||
| from typing import List, Optional, Set, Tuple | ||
|
|
||
| import asyncclick as click | ||
| from connector_ops.utils import ConnectorLanguage, SupportLevelEnum, get_all_connectors_in_repo # type: ignore | ||
| from pipelines import main_logger | ||
| from pipelines.cli.click_decorators import click_append_to_context_object, click_ignore_unused_kwargs, click_merge_args_into_context_obj | ||
| from pipelines.cli.lazy_group import LazyGroup | ||
| from pipelines.helpers.connectors.modifed import ConnectorWithModifiedFiles, get_connector_modified_files, get_modified_connectors | ||
| from pipelines.helpers.git import get_modified_files | ||
| from pipelines.helpers.utils import transform_strs_to_paths | ||
|
|
||
| ALL_CONNECTORS = get_all_connectors_in_repo() | ||
|
|
||
|
|
||
| def log_selected_connectors(selected_connectors_with_modified_files: List[ConnectorWithModifiedFiles]) -> None: | ||
| if selected_connectors_with_modified_files: | ||
| selected_connectors_names = [c.technical_name for c in selected_connectors_with_modified_files] | ||
| main_logger.info(f"Will run on the following {len(selected_connectors_names)} connectors: {', '.join(selected_connectors_names)}.") | ||
| else: | ||
| main_logger.info("No connectors to run.") | ||
|
|
||
| def get_selected_connectors_with_modified_files( | ||
| selected_names: Tuple[str], | ||
| selected_support_levels: Tuple[str], | ||
| selected_languages: Tuple[str], | ||
| modified: bool, | ||
| metadata_changes_only: bool, | ||
| with_changelog_entry_files: bool, | ||
| metadata_query: str, | ||
| modified_files: Set[Path], | ||
| enable_dependency_scanning: bool = False, | ||
| ) -> List[ConnectorWithModifiedFiles]: | ||
| """Get the connectors that match the selected criteria. | ||
| Args: | ||
| selected_names (Tuple[str]): Selected connector names. | ||
| selected_support_levels (Tuple[str]): Selected connector support levels. | ||
| selected_languages (Tuple[str]): Selected connector languages. | ||
| modified (bool): Whether to select the modified connectors. | ||
| metadata_changes_only (bool): Whether to select only the connectors with metadata changes. | ||
| with_changelog_entry_files (bool): Whether to select the connectors with files in .changelog_entries | ||
| modified_files (Set[Path]): The modified files. | ||
| enable_dependency_scanning (bool): Whether to enable the dependency scanning. | ||
| Returns: | ||
| List[ConnectorWithModifiedFiles]: The connectors that match the selected criteria. | ||
| """ | ||
|
|
||
| if metadata_changes_only and not modified: | ||
| main_logger.info("--metadata-changes-only overrides --modified") | ||
| modified = True | ||
|
|
||
| selected_modified_connectors = ( | ||
| get_modified_connectors(modified_files, ALL_CONNECTORS, enable_dependency_scanning) if modified else set() | ||
| ) | ||
| selected_connectors_by_name = {c for c in ALL_CONNECTORS if c.technical_name in selected_names} | ||
| selected_connectors_by_support_level = {connector for connector in ALL_CONNECTORS if connector.support_level in selected_support_levels} | ||
| selected_connectors_by_language = {connector for connector in ALL_CONNECTORS if connector.language in selected_languages} | ||
| selected_connectors_by_query = ( | ||
| {connector for connector in ALL_CONNECTORS if connector.metadata_query_match(metadata_query)} if metadata_query else set() | ||
| ) | ||
| selected_connectors_by_changelog_entry_files = {c for c in ALL_CONNECTORS if c.changelog_entry_files} if with_changelog_entry_files else set() | ||
|
|
||
| non_empty_connector_sets = [ | ||
| connector_set | ||
| for connector_set in [ | ||
| selected_connectors_by_name, | ||
| selected_connectors_by_support_level, | ||
| selected_connectors_by_language, | ||
| selected_connectors_by_query, | ||
| selected_modified_connectors, | ||
| selected_connectors_by_changelog_entry_files | ||
| ] | ||
| if connector_set | ||
| ] | ||
| # The selected connectors are the intersection of the selected connectors by name, support_level, language, simpleeval query and modified. | ||
| selected_connectors = set.intersection(*non_empty_connector_sets) if non_empty_connector_sets else set() | ||
|
|
||
| selected_connectors_with_modified_files = [] | ||
| for connector in selected_connectors: | ||
| connector_with_modified_files = ConnectorWithModifiedFiles( | ||
| relative_connector_path=connector.relative_connector_path, | ||
| modified_files=get_connector_modified_files(connector, modified_files), | ||
| ) | ||
| if not metadata_changes_only: | ||
| selected_connectors_with_modified_files.append(connector_with_modified_files) | ||
| else: | ||
| if connector_with_modified_files.has_metadata_change: | ||
| selected_connectors_with_modified_files.append(connector_with_modified_files) | ||
| return selected_connectors_with_modified_files | ||
|
|
||
|
|
||
| def validate_environment(is_local: bool) -> None: | ||
| """Check if the required environment variables exist.""" | ||
| if is_local: | ||
| if not Path(".git").is_dir(): | ||
| raise click.UsageError("You need to run this command from the repository root.") | ||
| else: | ||
| required_env_vars_for_ci = [ | ||
| "GCP_GSM_CREDENTIALS", | ||
| "CI_REPORT_BUCKET_NAME", | ||
| "CI_GITHUB_ACCESS_TOKEN", | ||
| "DOCKER_HUB_USERNAME", | ||
| "DOCKER_HUB_PASSWORD", | ||
| ] | ||
| for required_env_var in required_env_vars_for_ci: | ||
| if os.getenv(required_env_var) is None: | ||
| raise click.UsageError(f"When running in a CI context a {required_env_var} environment variable must be set.") | ||
|
|
||
|
|
||
| def should_use_remote_secrets(use_remote_secrets: Optional[bool]) -> bool: | ||
| """Check if the connector secrets should be loaded from Airbyte GSM or from the local secrets directory. | ||
| Args: | ||
| use_remote_secrets (Optional[bool]): Whether to use remote connector secrets or local connector secrets according to user inputs. | ||
| Raises: | ||
| click.UsageError: If the --use-remote-secrets flag was provided but no GCP_GSM_CREDENTIALS environment variable was found. | ||
| Returns: | ||
| bool: Whether to use remote connector secrets (True) or local connector secrets (False). | ||
| """ | ||
| gcp_gsm_credentials_is_set = bool(os.getenv("GCP_GSM_CREDENTIALS")) | ||
| if use_remote_secrets is None: | ||
| if gcp_gsm_credentials_is_set: | ||
| main_logger.info("GCP_GSM_CREDENTIALS environment variable found, using remote connector secrets.") | ||
| return True | ||
| else: | ||
| main_logger.info("No GCP_GSM_CREDENTIALS environment variable found, using local connector secrets.") | ||
| return False | ||
| if use_remote_secrets: | ||
| if gcp_gsm_credentials_is_set: | ||
| main_logger.info("GCP_GSM_CREDENTIALS environment variable found, using remote connector secrets.") | ||
| return True | ||
| else: | ||
| raise click.UsageError("The --use-remote-secrets flag was provided but no GCP_GSM_CREDENTIALS environment variable was found.") | ||
| else: | ||
| main_logger.info("Using local connector secrets as the --use-local-secrets flag was provided") | ||
| return False | ||
|
|
||
|
|
||
| @click.group( | ||
| cls=LazyGroup, | ||
| help="Commands related to connectors and connector acceptance tests.", | ||
| lazy_subcommands={ | ||
| "bump_version": "pipelines.airbyte_ci.cdk_java.bump_version.commands.bump_version", | ||
| }, | ||
| ) | ||
| @click_merge_args_into_context_obj | ||
| @click_append_to_context_object("use_remote_secrets", lambda ctx: should_use_remote_secrets(ctx.obj["use_remote_secrets"])) | ||
| @click.pass_context | ||
| @click_ignore_unused_kwargs | ||
| async def connectors( | ||
| ctx: click.Context, | ||
| ) -> None: | ||
| """Group all the connectors-ci command.""" | ||
| validate_environment(ctx.obj["is_local"]) | ||
|
|
||
| modified_files = [] | ||
| if ctx.obj["modified"] or ctx.obj["metadata_changes_only"]: | ||
| modified_files = transform_strs_to_paths( | ||
| await get_modified_files( | ||
| ctx.obj["git_branch"], | ||
| ctx.obj["git_revision"], | ||
| ctx.obj["diffed_branch"], | ||
| ctx.obj["is_local"], | ||
| ctx.obj["ci_context"], | ||
| ) | ||
| ) | ||
|
|
||
| ctx.obj["selected_connectors_with_modified_files"] = get_selected_connectors_with_modified_files( | ||
| ctx.obj["names"], | ||
| ctx.obj["support_levels"], | ||
| ctx.obj["languages"], | ||
| ctx.obj["modified"], | ||
| ctx.obj["metadata_changes_only"], | ||
| ctx.obj["with_changelog_entry_files"], | ||
| ctx.obj["metadata_query"], | ||
| set(modified_files), | ||
| ctx.obj["enable_dependency_scanning"], | ||
| ) | ||
| log_selected_connectors(ctx.obj["selected_connectors_with_modified_files"]) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters