Update Pipelines folder structure (airbytehq#31525)

Co-authored-by: Ben Church <ben@airbyte.io>
Co-authored-by: erohmensing <erohmensing@users.noreply.github.com>
Co-authored-by: bnchrch <bnchrch@users.noreply.github.com>

airbyte-ci/connectors/metadata_service/lib/tests/test_spec_cache.py

@@ -1,3 +1,7 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
 from unittest.mock import patch
 
 import pytest

airbyte-ci/connectors/pipelines/.gitignore

@@ -1 +1,2 @@
-pipeline_reports
+pipeline_reports
+.venv

airbyte-ci/connectors/pipelines/README.md

@@ -397,8 +397,9 @@ This command runs the Python tests for a airbyte-ci poetry package.
 
 ## Changelog
 | Version | PR                                                         | Description                                                                                               |
-| ------- | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- |
-| 2.0.2  | [#31533](https://github.com/airbytehq/airbyte/pull/31533)  | Pip cache volume by python version.                                                            |
+|---------| ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- |
+| 2.0.3   | [#31525](https://github.com/airbytehq/airbyte/pull/31525)  | Refactor folder structure                                                                                 |
+| 2.0.2   | [#31533](https://github.com/airbytehq/airbyte/pull/31533)  | Pip cache volume by python version.                                                            |
 | 2.0.1   | [#31545](https://github.com/airbytehq/airbyte/pull/31545)  | Reword the changelog entry when using `migrate-to-base-image`.                                                            |
 | 2.0.0   | [#31424](https://github.com/airbytehq/airbyte/pull/31424)  | Remove `airbyte-ci connectors format` command.                                                            |
 | 1.9.4   | [#31478](https://github.com/airbytehq/airbyte/pull/31478)  | Fix running tests for connector-ops package.                                                              |

airbyte-ci/connectors/pipelines/pipelines/__init__.py

@@ -8,7 +8,7 @@
 
 from rich.logging import RichHandler
 
-from . import sentry_utils
+from .helpers import sentry_utils
 
 sentry_utils.initialize()
 

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py

@@ -0,0 +1,58 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import anyio
+import click
+from pipelines.airbyte_ci.connectors.build_image.steps import run_connector_build_pipeline
+from pipelines.airbyte_ci.connectors.context import ConnectorContext
+from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines
+from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand
+
+
+@click.command(cls=DaggerPipelineCommand, help="Build all images for the selected connectors.")
+@click.option(
+    "--use-host-gradle-dist-tar",
+    is_flag=True,
+    help="Use gradle distTar output from host for java connectors.",
+    default=False,
+    type=bool,
+)
+@click.pass_context
+def build(ctx: click.Context, use_host_gradle_dist_tar: bool) -> bool:
+    """Runs a build pipeline for the selected connectors."""
+
+    connectors_contexts = [
+        ConnectorContext(
+            pipeline_name=f"Build connector {connector.technical_name}",
+            connector=connector,
+            is_local=ctx.obj["is_local"],
+            git_branch=ctx.obj["git_branch"],
+            git_revision=ctx.obj["git_revision"],
+            ci_report_bucket=ctx.obj["ci_report_bucket_name"],
+            report_output_prefix=ctx.obj["report_output_prefix"],
+            use_remote_secrets=ctx.obj["use_remote_secrets"],
+            gha_workflow_run_url=ctx.obj.get("gha_workflow_run_url"),
+            dagger_logs_url=ctx.obj.get("dagger_logs_url"),
+            pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"),
+            ci_context=ctx.obj.get("ci_context"),
+            ci_gcs_credentials=ctx.obj["ci_gcs_credentials"],
+            use_local_cdk=ctx.obj.get("use_local_cdk"),
+            open_report_in_browser=ctx.obj.get("open_report_in_browser"),
+            use_host_gradle_dist_tar=use_host_gradle_dist_tar,
+        )
+        for connector in ctx.obj["selected_connectors_with_modified_files"]
+    ]
+    if use_host_gradle_dist_tar and not ctx.obj["is_local"]:
+        raise Exception("flag --use-host-gradle-dist-tar requires --is-local")
+    anyio.run(
+        run_connectors_pipelines,
+        connectors_contexts,
+        run_connector_build_pipeline,
+        "Build Pipeline",
+        ctx.obj["concurrency"],
+        ctx.obj["dagger_logs_path"],
+        ctx.obj["execute_timeout"],
+    )
+
+    return True

airbyte-ci/connectors/pipelines/pipelines/builds/__init__.py → airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/__init__.py

@@ -9,11 +9,13 @@
 
 import anyio
 from connector_ops.utils import ConnectorLanguage
-from pipelines.bases import ConnectorReport, StepResult
-from pipelines.builds import java_connectors, python_connectors
-from pipelines.builds.common import LoadContainerToLocalDockerHost, StepStatus
+from pipelines.models.steps import StepResult
+from pipelines.airbyte_ci.connectors.build_image.steps import python_connectors
+from pipelines.airbyte_ci.connectors.build_image.steps.common import LoadContainerToLocalDockerHost, StepStatus
 from pipelines.consts import LOCAL_BUILD_PLATFORM
-from pipelines.contexts import ConnectorContext
+from pipelines.airbyte_ci.connectors.build_image.steps import java_connectors
+from pipelines.airbyte_ci.connectors.context import ConnectorContext
+from pipelines.airbyte_ci.connectors.reports import ConnectorReport
 
 
 class NoBuildStepForLanguageError(Exception):

airbyte-ci/connectors/pipelines/pipelines/builds/build_customization.py → airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/build_customization.py

@@ -1,6 +1,7 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
+
 import importlib
 from logging import Logger
 from types import ModuleType

airbyte-ci/connectors/pipelines/pipelines/builds/common.py → airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/common.py

@@ -7,10 +7,10 @@
 
 import docker
 from dagger import Container, ExecError, Platform, QueryError
-from pipelines.bases import Step, StepResult, StepStatus
+from pipelines.airbyte_ci.connectors.context import ConnectorContext
 from pipelines.consts import BUILD_PLATFORMS
-from pipelines.contexts import ConnectorContext
-from pipelines.utils import export_container_to_tarball
+from pipelines.helpers.utils import export_container_to_tarball
+from pipelines.models.steps import Step, StepResult, StepStatus
 
 
 class BuildConnectorImagesBase(Step, ABC):

airbyte-ci/connectors/pipelines/pipelines/builds/java_connectors.py → airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/java_connectors.py

@@ -5,12 +5,12 @@
 from typing import List, Optional, Tuple, Union
 
 from dagger import Container, Directory, ExecError, File, Host, Platform, QueryError
-from pipelines.actions import environments
-from pipelines.bases import StepResult, StepStatus
-from pipelines.builds.common import BuildConnectorImagesBase
+from pipelines.airbyte_ci.connectors.build_image.steps.common import BuildConnectorImagesBase
+from pipelines.airbyte_ci.connectors.context import ConnectorContext
+from pipelines.airbyte_ci.steps.gradle import GradleTask
 from pipelines.consts import LOCAL_BUILD_PLATFORM
-from pipelines.contexts import ConnectorContext
-from pipelines.gradle import GradleTask
+from pipelines.dagger.containers import java
+from pipelines.models.steps import StepResult, StepStatus
 
 
 class BuildConnectorDistributionTar(GradleTask):
@@ -46,7 +46,7 @@ async def _run(self, dist_dir: Directory) -> StepResult:
         return await super()._run(dist_tar)
 
     async def _build_connector(self, platform: Platform, dist_tar: File) -> Container:
-        return await environments.with_airbyte_java_connector(self.context, dist_tar, platform)
+        return await java.with_airbyte_java_connector(self.context, dist_tar, platform)
 
 
 async def run_connector_build(context: ConnectorContext) -> StepResult:

airbyte-ci/connectors/pipelines/pipelines/builds/normalization.py → airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/normalization.py

@@ -3,9 +3,9 @@
 #
 
 from dagger import Platform
-from pipelines.actions import environments
-from pipelines.bases import Step, StepResult, StepStatus
-from pipelines.contexts import ConnectorContext
+from pipelines.airbyte_ci.connectors.context import ConnectorContext
+from pipelines.dagger.actions.connector import normalization
+from pipelines.models.steps import Step, StepResult, StepStatus
 
 
 # TODO this class could be deleted
@@ -28,7 +28,7 @@ def __init__(self, context: ConnectorContext, normalization_image: str, build_pl
 
     async def _run(self) -> StepResult:
         if self.use_dev_normalization:
-            build_normalization_container = environments.with_normalization(self.context, self.build_platform)
+            build_normalization_container = normalization.with_normalization(self.context, self.build_platform)
         else:
             build_normalization_container = self.context.dagger_client.container().from_(self.normalization_image)
         return StepResult(self, StepStatus.SUCCESS, output_artifact=build_normalization_container)

airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py → airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/python_connectors.py

@@ -4,11 +4,11 @@
 
 
 from dagger import Container, Platform
-from pipelines.actions.environments import apply_python_development_overrides, with_python_connector_installed
-from pipelines.bases import StepResult
-from pipelines.builds import build_customization
-from pipelines.builds.common import BuildConnectorImagesBase
-from pipelines.contexts import ConnectorContext
+from pipelines.airbyte_ci.connectors.build_image.steps import build_customization
+from pipelines.airbyte_ci.connectors.build_image.steps.common import BuildConnectorImagesBase
+from pipelines.airbyte_ci.connectors.context import ConnectorContext
+from pipelines.dagger.actions.python.common import apply_python_development_overrides, with_python_connector_installed
+from pipelines.models.steps import StepResult
 
 
 class BuildConnectorImages(BuildConnectorImagesBase):

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/__init__.py

@@ -0,0 +1,3 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/commands.py

@@ -0,0 +1,61 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import anyio
+import click
+from pipelines.airbyte_ci.connectors.bump_version.pipeline import run_connector_version_bump_pipeline
+from pipelines.airbyte_ci.connectors.context import ConnectorContext
+from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines
+from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand
+
+
+@click.command(cls=DaggerPipelineCommand, short_help="Bump a connector version: update metadata.yaml and changelog.")
+@click.argument("bump-type", type=click.Choice(["patch", "minor", "major"]))
+@click.argument("pull-request-number", type=str)
+@click.argument("changelog-entry", type=str)
+@click.pass_context
+def bump_version(
+    ctx: click.Context,
+    bump_type: str,
+    pull_request_number: str,
+    changelog_entry: str,
+) -> bool:
+    """Bump a connector version: update metadata.yaml and changelog."""
+
+    connectors_contexts = [
+        ConnectorContext(
+            pipeline_name=f"Upgrade base image versions of connector {connector.technical_name}",
+            connector=connector,
+            is_local=ctx.obj["is_local"],
+            git_branch=ctx.obj["git_branch"],
+            git_revision=ctx.obj["git_revision"],
+            ci_report_bucket=ctx.obj["ci_report_bucket_name"],
+            report_output_prefix=ctx.obj["report_output_prefix"],
+            use_remote_secrets=ctx.obj["use_remote_secrets"],
+            gha_workflow_run_url=ctx.obj.get("gha_workflow_run_url"),
+            dagger_logs_url=ctx.obj.get("dagger_logs_url"),
+            pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"),
+            ci_context=ctx.obj.get("ci_context"),
+            ci_gcs_credentials=ctx.obj["ci_gcs_credentials"],
+            ci_git_user=ctx.obj["ci_git_user"],
+            ci_github_access_token=ctx.obj["ci_github_access_token"],
+            open_report_in_browser=False,
+        )
+        for connector in ctx.obj["selected_connectors_with_modified_files"]
+    ]
+
+    anyio.run(
+        run_connectors_pipelines,
+        connectors_contexts,
+        run_connector_version_bump_pipeline,
+        "Version bump pipeline pipeline",
+        ctx.obj["concurrency"],
+        ctx.obj["dagger_logs_path"],
+        ctx.obj["execute_timeout"],
+        bump_type,
+        changelog_entry,
+        pull_request_number,
+    )
+
+    return True