diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index c9b1148a24c1..a13dd5c7b8c8 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -81,7 +81,6 @@ At this point you can run `airbyte-ci` commands from the root of the repository. - [`connectors` command subgroup](#connectors-command-subgroup) * [Options](#options-1) - [`connectors list` command](#connectors-list-command) -- [`connectors format` command](#connectors-format-command) - [`connectors test` command](#connectors-test-command) * [Examples](#examples-) * [What it runs](#what-it-runs-) @@ -171,32 +170,6 @@ List connectors with multiple filters: `airbyte-ci connectors --language=low-code --support-level=certified list` -### `connectors format` command -Run a code formatter on one or multiple connectors. - -For Python connectors we run the following tools, using the configuration defined in `pyproject.toml`: -* `black` for code formatting -* `isort` for import sorting -* `licenseheaders` for adding license headers - -For Java connectors we run `./gradlew format`. - -In local CLI execution the formatted code is exported back the local repository. No commit or push is performed. -In CI execution the formatted code is pushed to the remote branch. One format commit per connector. - -#### Examples -Format a specific connector: - -`airbyte-ci connectors --name=source-pokeapi format` - -Format all Python connectors: - -`airbyte-ci connectors --language=python format` - -Format all connectors modified on the current branch: - -`airbyte-ci connectors --modified format` - ### `connectors test` command Run a test pipeline for one or multiple connectors. @@ -382,9 +355,9 @@ Make a connector using a Dockerfile migrate to the base image by: Migrate source-openweather to use the base image: `airbyte-ci connectors --name=source-openweather migrate-to-base-image` ### Arguments -| Argument | Description | -| --------------------- | ---------------------------------------------------------------------- | -| `PULL_REQUEST_NUMBER` | The GitHub pull request number, used in the changelog entry | +| Argument | Description | +| --------------------- | ----------------------------------------------------------- | +| `PULL_REQUEST_NUMBER` | The GitHub pull request number, used in the changelog entry | ### `metadata` command subgroup @@ -424,7 +397,8 @@ This command runs the Python tests for a airbyte-ci poetry package. ## Changelog | Version | PR | Description | -|---------| ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | +| ------- | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | +| 2.0.0 | [#31424](https://github.com/airbytehq/airbyte/pull/31424) | Remove `airbyte-ci connectors format` command. | | 1.9.4 | [#31478](https://github.com/airbytehq/airbyte/pull/31478) | Fix running tests for connector-ops package. | | 1.9.3 | [#31457](https://github.com/airbytehq/airbyte/pull/31457) | Improve the connector documentation for connectors migrated to our base image. | | 1.9.2 | [#31426](https://github.com/airbytehq/airbyte/pull/31426) | Concurrent execution of java connectors tests. | diff --git a/airbyte-ci/connectors/pipelines/pipelines/commands/groups/connectors.py b/airbyte-ci/connectors/pipelines/pipelines/commands/groups/connectors.py index 88a74793866e..f1521ba00af7 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/commands/groups/connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/commands/groups/connectors.py @@ -21,7 +21,6 @@ ) from pipelines.connector_changes.version_bump import run_connector_version_bump_pipeline from pipelines.contexts import ConnectorContext, ContextState, PublishConnectorContext -from pipelines.format import run_connector_format_pipeline from pipelines.github import update_global_commit_status_check_for_tests from pipelines.pipelines.connectors import run_connectors_pipelines from pipelines.publish import reorder_contexts, run_connector_publish_pipeline @@ -515,45 +514,6 @@ def list( return True -@connectors.command(name="format", cls=DaggerPipelineCommand, help="Autoformat connector code.") -@click.pass_context -def format_code(ctx: click.Context) -> bool: - connectors_contexts = [ - ConnectorContext( - pipeline_name=f"Format connector {connector.technical_name}", - connector=connector, - is_local=ctx.obj["is_local"], - git_branch=ctx.obj["git_branch"], - git_revision=ctx.obj["git_revision"], - ci_report_bucket=ctx.obj["ci_report_bucket_name"], - report_output_prefix=ctx.obj["report_output_prefix"], - use_remote_secrets=ctx.obj["use_remote_secrets"], - gha_workflow_run_url=ctx.obj.get("gha_workflow_run_url"), - dagger_logs_url=ctx.obj.get("dagger_logs_url"), - pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"), - ci_context=ctx.obj.get("ci_context"), - ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], - ci_git_user=ctx.obj["ci_git_user"], - ci_github_access_token=ctx.obj["ci_github_access_token"], - pull_request=ctx.obj.get("pull_request"), - should_save_report=False, - ) - for connector in ctx.obj["selected_connectors_with_modified_files"] - ] - - anyio.run( - run_connectors_pipelines, - connectors_contexts, - run_connector_format_pipeline, - "Format connectors pipeline", - ctx.obj["concurrency"], - ctx.obj["dagger_logs_path"], - ctx.obj["execute_timeout"], - ) - - return True - - @connectors.command(cls=DaggerPipelineCommand, help="Upgrades the base image version used by the selected connectors..") @click.option("--set-if-not-exists", default=True) @click.option( diff --git a/airbyte-ci/connectors/pipelines/pipelines/format/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/format/__init__.py deleted file mode 100644 index 730874bd6b9b..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/format/__init__.py +++ /dev/null @@ -1,101 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# -"""This module groups factory like functions to dispatch formatting steps according to the connector language.""" - -from __future__ import annotations - -import sys -from typing import List, Optional - -import anyio -import dagger -from connector_ops.utils import ConnectorLanguage -from pipelines.actions import environments -from pipelines.bases import ConnectorReport, Step, StepResult, StepStatus -from pipelines.contexts import ConnectorContext -from pipelines.format import java_connectors, python_connectors -from pipelines.git import GitPushChanges -from pipelines.pipelines.connectors import run_report_complete_pipeline - - -class NoFormatStepForLanguageError(Exception): - pass - - -FORMATTING_STEP_TO_CONNECTOR_LANGUAGE_MAPPING = { - ConnectorLanguage.PYTHON: python_connectors.FormatConnectorCode, - ConnectorLanguage.LOW_CODE: python_connectors.FormatConnectorCode, - ConnectorLanguage.JAVA: java_connectors.FormatConnectorCode, -} - - -class ExportChanges(Step): - title = "Export changes to local repository" - - async def _run(self, changed_directory: dagger.Directory, changed_directory_path_in_repo: str) -> StepResult: - await changed_directory.export(changed_directory_path_in_repo) - return StepResult(self, StepStatus.SUCCESS, stdout=f"Changes exported to {changed_directory_path_in_repo}") - - -async def run_connector_format_pipeline(context: ConnectorContext) -> ConnectorReport: - """Run a format pipeline for a single connector. - - Args: - context (ConnectorContext): The initialized connector context. - - Returns: - ConnectorReport: The reports holding formats results. - """ - steps_results = [] - async with context: - FormatConnectorCode = FORMATTING_STEP_TO_CONNECTOR_LANGUAGE_MAPPING.get(context.connector.language) - if not FormatConnectorCode: - raise NoFormatStepForLanguageError( - f"No formatting step found for connector {context.connector.technical_name} with language {context.connector.language}" - ) - format_connector_code_result = await FormatConnectorCode(context).run() - steps_results.append(format_connector_code_result) - - if context.is_local: - export_changes_results = await ExportChanges(context).run( - format_connector_code_result.output_artifact, str(context.connector.code_directory) - ) - steps_results.append(export_changes_results) - else: - git_push_changes_results = await GitPushChanges(context).run( - format_connector_code_result.output_artifact, - str(context.connector.code_directory), - f"Auto format {context.connector.technical_name} code", - skip_ci=True, - ) - steps_results.append(git_push_changes_results) - context.report = ConnectorReport(context, steps_results, name="FORMAT RESULTS") - return context.report - - -async def run_connectors_format_pipelines( - contexts: List[ConnectorContext], - ci_git_user: str, - ci_github_access_token: str, - git_branch: str, - is_local: bool, - execute_timeout: Optional[int], -) -> List[ConnectorContext]: - async with dagger.Connection(dagger.Config(log_output=sys.stderr, execute_timeout=execute_timeout)) as dagger_client: - requires_dind = any(context.connector.language == ConnectorLanguage.JAVA for context in contexts) - dockerd_service = environments.with_global_dockerd_service(dagger_client) - async with anyio.create_task_group() as tg_main: - if requires_dind: - tg_main.start_soon(dockerd_service.sync) - await anyio.sleep(10) # Wait for the docker service to be ready - for context in contexts: - context.dagger_client = dagger_client.pipeline(f"Format - {context.connector.technical_name}") - context.dockerd_service = dockerd_service - await run_connector_format_pipeline(context) - # When the connectors pipelines are done, we can stop the dockerd service - tg_main.cancel_scope.cancel() - - await run_report_complete_pipeline(dagger_client, contexts) - - return contexts diff --git a/airbyte-ci/connectors/pipelines/pipelines/format/java_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/format/java_connectors.py deleted file mode 100644 index aba30683fc41..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/format/java_connectors.py +++ /dev/null @@ -1,28 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from pipelines.actions import environments -from pipelines.bases import StepResult -from pipelines.gradle import GradleTask -from pipelines.utils import get_exec_result - - -class FormatConnectorCode(GradleTask): - """ - A step to format a Java connector code. - """ - - title = "Format connector code" - gradle_task_name = "format" - - async def _run(self) -> StepResult: - result = await super()._run() - return StepResult( - self, - result.status, - stderr=result.stderr, - stdout=result.stdout, - output_artifact=result.output_artifact.directory(str(self.context.connector.code_directory)), - ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/format/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/format/python_connectors.py deleted file mode 100644 index a6ba2a8e39cf..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/format/python_connectors.py +++ /dev/null @@ -1,57 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import asyncer -from pipelines.actions import environments -from pipelines.bases import Step, StepResult -from pipelines.utils import with_exit_code, with_stderr, with_stdout - - -class FormatConnectorCode(Step): - """ - A step to format a Python connector code. - """ - - title = "Format connector code" - - @property - def black_cmd(self): - return ["python", "-m", "black", f"--config=/{environments.PYPROJECT_TOML_FILE_PATH}", "."] - - @property - def isort_cmd(self): - return ["python", "-m", "isort", f"--settings-file=/{environments.PYPROJECT_TOML_FILE_PATH}", "."] - - @property - def licenseheaders_cmd(self): - return [ - "python", - "-m", - "licenseheaders", - f"--tmpl=/{environments.LICENSE_SHORT_FILE_PATH}", - "--ext=py", - "--exclude=**/models/__init__.py", - ] - - async def _run(self) -> StepResult: - formatted = ( - environments.with_testing_dependencies(self.context) - .with_mounted_directory("/connector_code", await self.context.get_connector_dir()) - .with_workdir("/connector_code") - .with_exec(self.licenseheaders_cmd) - .with_exec(self.isort_cmd) - .with_exec(self.black_cmd) - ) - async with asyncer.create_task_group() as task_group: - soon_exit_code = task_group.soonify(with_exit_code)(formatted) - soon_stderr = task_group.soonify(with_stderr)(formatted) - soon_stdout = task_group.soonify(with_stdout)(formatted) - - return StepResult( - self, - self.get_step_status_from_exit_code(soon_exit_code.value), - stderr=soon_stderr.value, - stdout=soon_stdout.value, - output_artifact=formatted.directory("/connector_code"), - ) diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 1032ea6d3a9d..215724eb9871 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "1.9.4" +version = "2.0.0" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "] diff --git a/airbyte-ci/connectors/pipelines/tests/test_commands/test_groups/test_connectors.py b/airbyte-ci/connectors/pipelines/tests/test_commands/test_groups/test_connectors.py index e7bf1141845d..c7c0b6aa819b 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_commands/test_groups/test_connectors.py +++ b/airbyte-ci/connectors/pipelines/tests/test_commands/test_groups/test_connectors.py @@ -266,7 +266,6 @@ def click_context_obj(): "test", ], ), - (connectors.format_code, []), (connectors.build, []), ], )