From a02c0f892d98db3373a2210c1be15f9bda867fc3 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Wed, 10 Apr 2024 13:43:23 +0200 Subject: [PATCH] airbyte-ci: re-enable connector dependency upload on publish --- airbyte-ci/connectors/pipelines/README.md | 1 + .../airbyte_ci/connectors/publish/pipeline.py | 14 +++++++------- airbyte-ci/connectors/pipelines/pyproject.toml | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index dc846dd4cce37..2e38d39cc5258 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -649,6 +649,7 @@ E.G.: running Poe tasks on the modified internal packages of the current branch: | Version | PR | Description | | ------- | ---------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- | +| 4.7.2 | [#TBD](https://github.com/airbytehq/airbyte/pull/TBD) | Re-enable connector dependencies upload on publish. | | 4.7.1 | [#36961](https://github.com/airbytehq/airbyte/pull/36961) | Temporarily disable python connectors dependencies upload until we find a schema the data team can work with. | | 4.7.0 | [#36892](https://github.com/airbytehq/airbyte/pull/36892) | Upload Python connectors dependencies list to GCS on publish. | | 4.6.5 | [#36722](https://github.com/airbytehq/airbyte/pull/36527) | Fix incorrect pipeline names | diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py index 1192fd5277859..7cd7e4c0a45c1 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py @@ -84,7 +84,7 @@ class ConnectorDependenciesMetadata(BaseModel): connector_repository: str connector_version: str connector_definition_id: str - dependencies: Dict[str, str] + dependencies: List[Dict[str, str]] generation_time: datetime = datetime.utcnow() @@ -100,7 +100,9 @@ async def _run(self, built_containers_per_platform: Dict[Platform, Container]) - ], "This step can only run for Python connectors." built_container = built_containers_per_platform[LOCAL_BUILD_PLATFORM] pip_freeze_output = await built_container.with_exec(["pip", "freeze"], skip_entrypoint=True).stdout() - dependencies = {line.split("==")[0]: line.split("==")[1] for line in pip_freeze_output.splitlines() if "==" in line} + dependencies = [ + {"package_name": line.split("==")[0], "version": line.split("==")[1]} for line in pip_freeze_output.splitlines() if "==" in line + ] connector_technical_name = self.context.connector.technical_name connector_version = self.context.metadata["dockerImageTag"] dependencies_metadata = ConnectorDependenciesMetadata( @@ -365,11 +367,9 @@ def create_connector_report(results: List[StepResult]) -> ConnectorReport: if build_connector_results.status is not StepStatus.SUCCESS: return create_connector_report(results) - # Temporarily disabling it until we find the correct schema that the data team can ingest - - # if context.connector.language in [ConnectorLanguage.PYTHON, ConnectorLanguage.LOW_CODE]: - # upload_dependencies_step = await UploadDependenciesToMetadataService(context).run(build_connector_results.output) - # results.append(upload_dependencies_step) + if context.connector.language in [ConnectorLanguage.PYTHON, ConnectorLanguage.LOW_CODE]: + upload_dependencies_step = await UploadDependenciesToMetadataService(context).run(build_connector_results.output) + results.append(upload_dependencies_step) built_connector_platform_variants = list(build_connector_results.output.values()) push_connector_image_results = await PushConnectorImageToRegistry(context).run(built_connector_platform_variants) diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index b0501a7bbb193..6ca49e1c1e60f 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "4.7.1" +version = "4.7.2" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "]