airbyte-ci: implement migrate-to-poetry connectors command (#35583)

airbyte-ci/connectors/pipelines/README.md

@@ -145,6 +145,7 @@ At this point you can run `airbyte-ci` commands.
 - [`connectors upgrade_cdk` command](#connectors-upgrade_cdk)
 - [`connectors upgrade_base_image` command](#connectors-upgrade_base_image)
 - [`connectors migrate_to_base_image` command](#connectors-migrate_to_base_image)
+- [`connectors migrate-to-poetry` command](#connectors-migrate-to-poetry)
 - [`format` command subgroup](#format-subgroup)
   - [`format check` command](#format-check-command)
   - [`format fix` command](#format-fix-command)
@@ -512,16 +513,20 @@ Make a connector using a Dockerfile migrate to the base image by:
 - Updating its documentation to explain the build process
 - Bumping by a patch version
 
-### Examples
+#### Examples
 
 Migrate source-openweather to use the base image:
 `airbyte-ci connectors --name=source-openweather migrate_to_base_image`
 
-### Arguments
 
-| Argument              | Description                                                 |
-| --------------------- | ----------------------------------------------------------- |
-| `PULL_REQUEST_NUMBER` | The GitHub pull request number, used in the changelog entry |
+### <a id="connectors-migrate-to-poetry"></a>`connectors migrate-to-poetry` command
+
+Migrate connectors the poetry package manager.
+
+#### Examples
+
+Migrate source-openweather to use the base image:
+`airbyte-ci connectors --name=source-openweather migrate-to-poetry`
 
 ### <a id="format-subgroup"></a>`format` command subgroup
 
@@ -644,8 +649,9 @@ E.G.: running Poe tasks on the modified internal packages of the current branch:
 
 | Version | PR                                                         | Description                                                                                                                |
 | ------- | ---------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- |
-| 4.5.4   | [#36206](https://github.com/airbytehq/airbyte/pull/36206) | Revert poetry cache removal during nightly builds                                                                          |
-| 4.5.3   | [#34586](https://github.com/airbytehq/airbyte/pull/34586)  | Extract connector changelog modification logic into its own class                                                       |
+| 4.6.0   | [#35583](https://github.com/airbytehq/airbyte/pull/35583)  | Implement the `airbyte-ci connectors migrate-to-poetry` command.                                                           |
+| 4.5.4   | [#36206](https://github.com/airbytehq/airbyte/pull/36206)  | Revert poetry cache removal during nightly builds                                                                          |
+| 4.5.3   | [#34586](https://github.com/airbytehq/airbyte/pull/34586)  | Extract connector changelog modification logic into its own class                                                          |
 | 4.5.2   | [#35802](https://github.com/airbytehq/airbyte/pull/35802)  | Fix bug with connectors bump_version command                                                                               |
 | 4.5.1   | [#35786](https://github.com/airbytehq/airbyte/pull/35786)  | Declare `live_tests` as an internal poetry package.                                                                        |
 | 4.5.0   | [#35784](https://github.com/airbytehq/airbyte/pull/35784)  | Format command supports kotlin                                                                                             |

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/pipeline.py

@@ -42,12 +42,14 @@ def __init__(
         new_version: str,
         comment: str,
         pull_request_number: str,
+        export_docs: bool = False,
     ) -> None:
         super().__init__(context)
         self.repo_dir = repo_dir
         self.new_version = semver.VersionInfo.parse(new_version)
         self.comment = comment
         self.pull_request_number = int(pull_request_number)
+        self.export_docs = export_docs
 
     async def _run(self) -> StepResult:
         doc_path = self.context.connector.documentation_file_path
@@ -68,6 +70,8 @@ async def _run(self) -> StepResult:
                 step=self, status=StepStatus.FAILURE, stderr=f"Could not add changelog entry: {e}", output=self.repo_dir, exc_info=e
             )
         updated_repo_dir = self.repo_dir.with_new_file(str(doc_path), contents=updated_doc)
+        if self.export_docs:
+            await updated_repo_dir.file(str(doc_path)).export(str(doc_path))
         return StepResult(
             step=self,
             status=StepStatus.SUCCESS,
@@ -85,10 +89,12 @@ def __init__(
         context: ConnectorContext,
         repo_dir: Directory,
         new_version: str,
+        export_metadata: bool = False,
     ) -> None:
         super().__init__(context)
         self.repo_dir = repo_dir
         self.new_version = new_version
+        self.export_metadata = export_metadata
 
     @staticmethod
     def get_metadata_with_bumped_version(previous_version: str, new_version: str, metadata_str: str) -> str:
@@ -110,12 +116,13 @@ async def _run(self) -> StepResult:
         repo_dir_with_updated_metadata = metadata_change_helpers.get_repo_dir_with_updated_metadata_str(
             self.repo_dir, metadata_path, updated_metadata_str
         )
-
         metadata_validation_results = await MetadataValidation(self.context).run()
         # Exit early if the metadata file is invalid.
         if metadata_validation_results.status is not StepStatus.SUCCESS:
             return metadata_validation_results
 
+        if self.export_metadata:
+            await repo_dir_with_updated_metadata.file(str(metadata_path)).export(str(metadata_path))
         return StepResult(
             step=self,
             status=StepStatus.SUCCESS,

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py

@@ -151,6 +151,7 @@ def should_use_remote_secrets(use_remote_secrets: Optional[bool]) -> bool:
         "publish": "pipelines.airbyte_ci.connectors.publish.commands.publish",
         "bump_version": "pipelines.airbyte_ci.connectors.bump_version.commands.bump_version",
         "migrate_to_base_image": "pipelines.airbyte_ci.connectors.migrate_to_base_image.commands.migrate_to_base_image",
+        "migrate-to-poetry": "pipelines.airbyte_ci.connectors.migrate_to_poetry.commands.migrate_to_poetry",
         "upgrade_base_image": "pipelines.airbyte_ci.connectors.upgrade_base_image.commands.upgrade_base_image",
         "upgrade_cdk": "pipelines.airbyte_ci.connectors.upgrade_cdk.commands.bump_version",
     },

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/consts.py

@@ -23,6 +23,13 @@ class CONNECTOR_TEST_STEP_ID(str, Enum):
     VERSION_INC_CHECK = "version_inc_check"
     TEST_ORCHESTRATOR = "test_orchestrator"
     DEPLOY_ORCHESTRATOR = "deploy_orchestrator"
+    UPDATE_README = "update_readme"
+    ADD_CHANGELOG_ENTRY = "add_changelog_entry"
+    BUMP_METADATA_VERSION = "bump_metadata_version"
+    REGRESSION_TEST = "regression_test"
+    CHECK_MIGRATION_CANDIDATE = "check_migration_candidate"
+    POETRY_INIT = "poetry_init"
+    DELETE_SETUP_PY = "delete_setup_py"
 
     def __str__(self) -> str:
         return self.value

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_poetry/__init__.py

@@ -0,0 +1,3 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_poetry/commands.py

@@ -0,0 +1,57 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+
+import asyncclick as click
+from pipelines.airbyte_ci.connectors.context import ConnectorContext
+from pipelines.airbyte_ci.connectors.migrate_to_poetry.pipeline import run_connector_migration_to_poetry_pipeline
+from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines
+from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand
+
+
+@click.command(
+    cls=DaggerPipelineCommand,
+    short_help="Migrate the selected connectors to poetry.",
+)
+@click.pass_context
+async def migrate_to_poetry(
+    ctx: click.Context,
+) -> bool:
+
+    connectors_contexts = [
+        ConnectorContext(
+            pipeline_name=f"Migrate {connector.technical_name} to Poetry",
+            connector=connector,
+            is_local=ctx.obj["is_local"],
+            git_branch=ctx.obj["git_branch"],
+            git_revision=ctx.obj["git_revision"],
+            ci_report_bucket=ctx.obj["ci_report_bucket_name"],
+            report_output_prefix=ctx.obj["report_output_prefix"],
+            use_remote_secrets=ctx.obj["use_remote_secrets"],
+            gha_workflow_run_url=ctx.obj.get("gha_workflow_run_url"),
+            dagger_logs_url=ctx.obj.get("dagger_logs_url"),
+            pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"),
+            ci_context=ctx.obj.get("ci_context"),
+            ci_gcs_credentials=ctx.obj["ci_gcs_credentials"],
+            ci_git_user=ctx.obj["ci_git_user"],
+            ci_github_access_token=ctx.obj["ci_github_access_token"],
+            enable_report_auto_open=True,
+            docker_hub_username=ctx.obj.get("docker_hub_username"),
+            docker_hub_password=ctx.obj.get("docker_hub_password"),
+            s3_build_cache_access_key_id=ctx.obj.get("s3_build_cache_access_key_id"),
+            s3_build_cache_secret_key=ctx.obj.get("s3_build_cache_secret_key"),
+        )
+        for connector in ctx.obj["selected_connectors_with_modified_files"]
+    ]
+
+    await run_connectors_pipelines(
+        connectors_contexts,
+        run_connector_migration_to_poetry_pipeline,
+        "Migration to poetry pipeline",
+        ctx.obj["concurrency"],
+        ctx.obj["dagger_logs_path"],
+        ctx.obj["execute_timeout"],
+    )
+
+    return True