airbyte-ci: re-enable connector dependency upload on publish

airbytehq · Apr 10, 2024 · a02c0f8 · a02c0f8
1 parent 442177b
commit a02c0f8
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 8 deletions.
diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md
@@ -649,6 +649,7 @@ E.G.: running Poe tasks on the modified internal packages of the current branch:
 
 | Version | PR                                                         | Description                                                                                                                |
 | ------- | ---------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- |
+| 4.7.2   | [#TBD](https://github.com/airbytehq/airbyte/pull/TBD)      | Re-enable connector dependencies upload on publish.                                                                        |
 | 4.7.1   | [#36961](https://github.com/airbytehq/airbyte/pull/36961)  | Temporarily disable python connectors dependencies upload until we find a schema the data team can work with.              |
 | 4.7.0   | [#36892](https://github.com/airbytehq/airbyte/pull/36892)  | Upload Python connectors dependencies list to GCS on publish.                                                              |
 | 4.6.5   | [#36722](https://github.com/airbytehq/airbyte/pull/36527)  | Fix incorrect pipeline names                                                                                               |

diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py
@@ -84,7 +84,7 @@ class ConnectorDependenciesMetadata(BaseModel):
     connector_repository: str
     connector_version: str
     connector_definition_id: str
-    dependencies: Dict[str, str]
+    dependencies: List[Dict[str, str]]
     generation_time: datetime = datetime.utcnow()
 
 
@@ -100,7 +100,9 @@ async def _run(self, built_containers_per_platform: Dict[Platform, Container]) -
         ], "This step can only run for Python connectors."
         built_container = built_containers_per_platform[LOCAL_BUILD_PLATFORM]
         pip_freeze_output = await built_container.with_exec(["pip", "freeze"], skip_entrypoint=True).stdout()
-        dependencies = {line.split("==")[0]: line.split("==")[1] for line in pip_freeze_output.splitlines() if "==" in line}
+        dependencies = [
+            {"package_name": line.split("==")[0], "version": line.split("==")[1]} for line in pip_freeze_output.splitlines() if "==" in line
+        ]
         connector_technical_name = self.context.connector.technical_name
         connector_version = self.context.metadata["dockerImageTag"]
         dependencies_metadata = ConnectorDependenciesMetadata(
@@ -365,11 +367,9 @@ def create_connector_report(results: List[StepResult]) -> ConnectorReport:
             if build_connector_results.status is not StepStatus.SUCCESS:
                 return create_connector_report(results)
 
-            # Temporarily disabling it until we find the correct schema that the data team can ingest
-
-            # if context.connector.language in [ConnectorLanguage.PYTHON, ConnectorLanguage.LOW_CODE]:
-            #     upload_dependencies_step = await UploadDependenciesToMetadataService(context).run(build_connector_results.output)
-            #     results.append(upload_dependencies_step)
+            if context.connector.language in [ConnectorLanguage.PYTHON, ConnectorLanguage.LOW_CODE]:
+                upload_dependencies_step = await UploadDependenciesToMetadataService(context).run(build_connector_results.output)
+                results.append(upload_dependencies_step)
 
             built_connector_platform_variants = list(build_connector_results.output.values())
             push_connector_image_results = await PushConnectorImageToRegistry(context).run(built_connector_platform_variants)

diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "4.7.1"
+version = "4.7.2"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <contact@airbyte.io>"]