From 198622e4f0eb2588d50e73b3851b71a91b6fe4c3 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Mon, 23 Oct 2023 10:03:34 -0500 Subject: [PATCH 1/4] pass cdk into publish command --- .../pipelines/airbyte_ci/connectors/publish/commands.py | 1 + .../pipelines/airbyte_ci/connectors/publish/context.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py index adc8be6f101e..c5108126380c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py @@ -118,6 +118,7 @@ def publish( pull_request=ctx.obj.get("pull_request"), s3_build_cache_access_key_id=ctx.obj.get("s3_build_cache_access_key_id"), s3_build_cache_secret_key=ctx.obj.get("s3_build_cache_secret_key"), + use_local_cdk=ctx.obj["use_local_cdk"], ) for connector in ctx.obj["selected_connectors_with_modified_files"] ] diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py index 932c7e3a030f..c50bca2293e9 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py @@ -41,6 +41,7 @@ def __init__( pull_request: PullRequest = None, s3_build_cache_access_key_id: Optional[str] = None, s3_build_cache_secret_key: Optional[str] = None, + use_local_cdk: bool = False, ): self.pre_release = pre_release self.spec_cache_bucket_name = spec_cache_bucket_name @@ -66,6 +67,7 @@ def __init__( reporting_slack_channel=reporting_slack_channel, ci_gcs_credentials=ci_gcs_credentials, should_save_report=True, + use_local_cdk=use_local_cdk, docker_hub_username=docker_hub_username, docker_hub_password=docker_hub_password, s3_build_cache_access_key_id=s3_build_cache_access_key_id, From 9f51109c6bd565c63fdc6cf6daee96834c79e969 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Mon, 23 Oct 2023 11:08:15 -0500 Subject: [PATCH 2/4] validate options and increment version --- airbyte-ci/connectors/pipelines/README.md | 1 + .../pipelines/airbyte_ci/connectors/publish/context.py | 3 +++ airbyte-ci/connectors/pipelines/pyproject.toml | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index 37ca60014196..f86a45d70e28 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -398,6 +398,7 @@ This command runs the Python tests for a airbyte-ci poetry package. ## Changelog | Version | PR | Description | | ------- | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | +| 2.2.5 | [#31716](https://github.com/airbytehq/airbyte/pull/31716) | Enable pre-release publish with local CDK. | 2.2.4 | [#31535](https://github.com/airbytehq/airbyte/pull/31535) | Improve gradle caching when building java connectors. | | 2.2.3 | [#31688](https://github.com/airbytehq/airbyte/pull/31688) | Fix failing `CheckBaseImageUse` step when not running on PR. | | 2.2.2 | [#31659](https://github.com/airbytehq/airbyte/pull/31659) | Support builds on x86_64 platform | diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py index c50bca2293e9..d62ecb3d0e8c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py @@ -51,6 +51,9 @@ def __init__( pipeline_name = f"Publish {connector.technical_name}" pipeline_name = pipeline_name + " (pre-release)" if pre_release else pipeline_name + if use_local_cdk and not self.pre_release: + raise ValueError("Publishing with the local CDK is only supported for pre-release publishing.") + super().__init__( pipeline_name=pipeline_name, connector=connector, diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 4b2ff0ba8ef6..692624715fc9 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "2.2.4" +version = "2.2.5" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "] From a903d27afc912ed009a2772f64c82e55c1df823b Mon Sep 17 00:00:00 2001 From: Ella Rohm-Ensing Date: Mon, 23 Oct 2023 09:41:47 -0700 Subject: [PATCH 3/4] Apply suggestions from code review --- airbyte-ci/connectors/pipelines/README.md | 2 +- .../pipelines/airbyte_ci/connectors/publish/context.py | 2 +- airbyte-ci/connectors/pipelines/pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index f86a45d70e28..28b3489c7550 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -398,7 +398,7 @@ This command runs the Python tests for a airbyte-ci poetry package. ## Changelog | Version | PR | Description | | ------- | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | -| 2.2.5 | [#31716](https://github.com/airbytehq/airbyte/pull/31716) | Enable pre-release publish with local CDK. +| 2.3.0 | [#31716](https://github.com/airbytehq/airbyte/pull/31716) | Enable pre-release publish with local CDK. | 2.2.4 | [#31535](https://github.com/airbytehq/airbyte/pull/31535) | Improve gradle caching when building java connectors. | | 2.2.3 | [#31688](https://github.com/airbytehq/airbyte/pull/31688) | Fix failing `CheckBaseImageUse` step when not running on PR. | | 2.2.2 | [#31659](https://github.com/airbytehq/airbyte/pull/31659) | Support builds on x86_64 platform | diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py index d62ecb3d0e8c..1c66d2649cd3 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py @@ -52,7 +52,7 @@ def __init__( pipeline_name = pipeline_name + " (pre-release)" if pre_release else pipeline_name if use_local_cdk and not self.pre_release: - raise ValueError("Publishing with the local CDK is only supported for pre-release publishing.") + raise click.UsageError("Publishing with the local CDK is only supported for pre-release publishing.") super().__init__( pipeline_name=pipeline_name, diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 692624715fc9..3fe3a7eb34b5 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "2.2.5" +version = "2.3.0" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "] From 1e279ec27e379d4619172e7fddf2010733d452f6 Mon Sep 17 00:00:00 2001 From: Ella Rohm-Ensing Date: Tue, 24 Oct 2023 08:12:49 -0700 Subject: [PATCH 4/4] make param optional --- .../pipelines/airbyte_ci/connectors/publish/commands.py | 2 +- .../pipelines/airbyte_ci/connectors/publish/context.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py index 7e8af7bda1e1..6f50306475f2 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py @@ -105,7 +105,7 @@ def publish( pull_request=ctx.obj.get("pull_request"), s3_build_cache_access_key_id=ctx.obj.get("s3_build_cache_access_key_id"), s3_build_cache_secret_key=ctx.obj.get("s3_build_cache_secret_key"), - use_local_cdk=ctx.obj["use_local_cdk"], + use_local_cdk=ctx.obj.get("use_local_cdk"), ) for connector in ctx.obj["selected_connectors_with_modified_files"] ] diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py index 69f0b80d926c..633c19e17b3f 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py @@ -41,7 +41,7 @@ def __init__( pull_request: PullRequest = None, s3_build_cache_access_key_id: Optional[str] = None, s3_build_cache_secret_key: Optional[str] = None, - use_local_cdk: bool = False, + use_local_cdk: Optional[bool] = False, ): self.pre_release = pre_release self.spec_cache_bucket_name = spec_cache_bucket_name