forked from airbytehq/airbyte
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Update Pipelines folder structure (airbytehq#31525)
Co-authored-by: Ben Church <ben@airbyte.io> Co-authored-by: erohmensing <erohmensing@users.noreply.github.com> Co-authored-by: bnchrch <bnchrch@users.noreply.github.com>
- Loading branch information
Showing
113 changed files
with
4,341 additions
and
4,006 deletions.
There are no files selected for viewing
4 changes: 4 additions & 0 deletions
4
airbyte-ci/connectors/metadata_service/lib/tests/test_spec_cache.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,3 +1,7 @@ | ||
# | ||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved. | ||
# | ||
|
||
from unittest.mock import patch | ||
|
||
import pytest | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1 +1,2 @@ | ||
pipeline_reports | ||
pipeline_reports | ||
.venv |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
4 changes: 0 additions & 4 deletions
4
airbyte-ci/connectors/pipelines/pipelines/actions/__init__.py
This file was deleted.
Oops, something went wrong.
1,015 changes: 0 additions & 1,015 deletions
1,015
airbyte-ci/connectors/pipelines/pipelines/actions/environments.py
This file was deleted.
Oops, something went wrong.
File renamed without changes.
File renamed without changes.
File renamed without changes.
58 changes: 58 additions & 0 deletions
58
airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,58 @@ | ||
# | ||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved. | ||
# | ||
|
||
import anyio | ||
import click | ||
from pipelines.airbyte_ci.connectors.build_image.steps import run_connector_build_pipeline | ||
from pipelines.airbyte_ci.connectors.context import ConnectorContext | ||
from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines | ||
from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand | ||
|
||
|
||
@click.command(cls=DaggerPipelineCommand, help="Build all images for the selected connectors.") | ||
@click.option( | ||
"--use-host-gradle-dist-tar", | ||
is_flag=True, | ||
help="Use gradle distTar output from host for java connectors.", | ||
default=False, | ||
type=bool, | ||
) | ||
@click.pass_context | ||
def build(ctx: click.Context, use_host_gradle_dist_tar: bool) -> bool: | ||
"""Runs a build pipeline for the selected connectors.""" | ||
|
||
connectors_contexts = [ | ||
ConnectorContext( | ||
pipeline_name=f"Build connector {connector.technical_name}", | ||
connector=connector, | ||
is_local=ctx.obj["is_local"], | ||
git_branch=ctx.obj["git_branch"], | ||
git_revision=ctx.obj["git_revision"], | ||
ci_report_bucket=ctx.obj["ci_report_bucket_name"], | ||
report_output_prefix=ctx.obj["report_output_prefix"], | ||
use_remote_secrets=ctx.obj["use_remote_secrets"], | ||
gha_workflow_run_url=ctx.obj.get("gha_workflow_run_url"), | ||
dagger_logs_url=ctx.obj.get("dagger_logs_url"), | ||
pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"), | ||
ci_context=ctx.obj.get("ci_context"), | ||
ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], | ||
use_local_cdk=ctx.obj.get("use_local_cdk"), | ||
open_report_in_browser=ctx.obj.get("open_report_in_browser"), | ||
use_host_gradle_dist_tar=use_host_gradle_dist_tar, | ||
) | ||
for connector in ctx.obj["selected_connectors_with_modified_files"] | ||
] | ||
if use_host_gradle_dist_tar and not ctx.obj["is_local"]: | ||
raise Exception("flag --use-host-gradle-dist-tar requires --is-local") | ||
anyio.run( | ||
run_connectors_pipelines, | ||
connectors_contexts, | ||
run_connector_build_pipeline, | ||
"Build Pipeline", | ||
ctx.obj["concurrency"], | ||
ctx.obj["dagger_logs_path"], | ||
ctx.obj["execute_timeout"], | ||
) | ||
|
||
return True |
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
1 change: 1 addition & 0 deletions
1
...s/pipelines/builds/build_customization.py → .../build_image/steps/build_customization.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
3 changes: 3 additions & 0 deletions
3
airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/__init__.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
# | ||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved. | ||
# |
61 changes: 61 additions & 0 deletions
61
airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/commands.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,61 @@ | ||
# | ||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved. | ||
# | ||
|
||
import anyio | ||
import click | ||
from pipelines.airbyte_ci.connectors.bump_version.pipeline import run_connector_version_bump_pipeline | ||
from pipelines.airbyte_ci.connectors.context import ConnectorContext | ||
from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines | ||
from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand | ||
|
||
|
||
@click.command(cls=DaggerPipelineCommand, short_help="Bump a connector version: update metadata.yaml and changelog.") | ||
@click.argument("bump-type", type=click.Choice(["patch", "minor", "major"])) | ||
@click.argument("pull-request-number", type=str) | ||
@click.argument("changelog-entry", type=str) | ||
@click.pass_context | ||
def bump_version( | ||
ctx: click.Context, | ||
bump_type: str, | ||
pull_request_number: str, | ||
changelog_entry: str, | ||
) -> bool: | ||
"""Bump a connector version: update metadata.yaml and changelog.""" | ||
|
||
connectors_contexts = [ | ||
ConnectorContext( | ||
pipeline_name=f"Upgrade base image versions of connector {connector.technical_name}", | ||
connector=connector, | ||
is_local=ctx.obj["is_local"], | ||
git_branch=ctx.obj["git_branch"], | ||
git_revision=ctx.obj["git_revision"], | ||
ci_report_bucket=ctx.obj["ci_report_bucket_name"], | ||
report_output_prefix=ctx.obj["report_output_prefix"], | ||
use_remote_secrets=ctx.obj["use_remote_secrets"], | ||
gha_workflow_run_url=ctx.obj.get("gha_workflow_run_url"), | ||
dagger_logs_url=ctx.obj.get("dagger_logs_url"), | ||
pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"), | ||
ci_context=ctx.obj.get("ci_context"), | ||
ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], | ||
ci_git_user=ctx.obj["ci_git_user"], | ||
ci_github_access_token=ctx.obj["ci_github_access_token"], | ||
open_report_in_browser=False, | ||
) | ||
for connector in ctx.obj["selected_connectors_with_modified_files"] | ||
] | ||
|
||
anyio.run( | ||
run_connectors_pipelines, | ||
connectors_contexts, | ||
run_connector_version_bump_pipeline, | ||
"Version bump pipeline pipeline", | ||
ctx.obj["concurrency"], | ||
ctx.obj["dagger_logs_path"], | ||
ctx.obj["execute_timeout"], | ||
bump_type, | ||
changelog_entry, | ||
pull_request_number, | ||
) | ||
|
||
return True |
Oops, something went wrong.