-
Notifications
You must be signed in to change notification settings - Fork 3.8k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
add the ability to upgrade CDK for java connectors #34343
Merged
stephane-airbyte
merged 2 commits into
master
from
stephane/01-17-add_the_ability_to_upgrade_CDK_for_java_connectors
Jan 25, 2024
Merged
Changes from all commits
Commits
File filter
Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -7,12 +7,17 @@ | |
import re | ||
from typing import TYPE_CHECKING | ||
|
||
from connector_ops.utils import ConnectorLanguage # type: ignore | ||
from dagger import Directory | ||
from pipelines.airbyte_ci.connectors.context import ConnectorContext | ||
from pipelines.airbyte_ci.connectors.reports import ConnectorReport, Report | ||
from pipelines.helpers import git | ||
from pipelines.helpers.connectors import cdk_helpers | ||
from pipelines.models.steps import Step, StepResult, StepStatus | ||
|
||
if TYPE_CHECKING: | ||
from typing import Optional | ||
|
||
from anyio import Semaphore | ||
|
||
|
||
|
@@ -30,14 +35,22 @@ def __init__( | |
|
||
async def _run(self) -> StepResult: | ||
context = self.context | ||
og_connector_dir = await context.get_connector_dir() | ||
if "setup.py" not in await og_connector_dir.entries(): | ||
return self.skip("Connector does not have a setup.py file.") | ||
setup_py = og_connector_dir.file("setup.py") | ||
setup_py_content = await setup_py.contents() | ||
|
||
try: | ||
updated_setup_py = self.update_cdk_version(setup_py_content) | ||
updated_connector_dir = og_connector_dir.with_new_file("setup.py", updated_setup_py) | ||
og_connector_dir = await context.get_connector_dir() | ||
if self.context.connector.language in [ConnectorLanguage.PYTHON, ConnectorLanguage.LOW_CODE]: | ||
updated_connector_dir = await self.upgrade_cdk_version_for_python_connector(og_connector_dir) | ||
elif self.context.connector.language is ConnectorLanguage.JAVA: | ||
updated_connector_dir = await self.upgrade_cdk_version_for_java_connector(og_connector_dir) | ||
else: | ||
return StepResult( | ||
self, | ||
StepStatus.FAILURE, | ||
stderr=f"No CDK for connector {self.context.connector.technical_name} of written in {self.context.connector.language}", | ||
) | ||
|
||
if updated_connector_dir is None: | ||
return self.skip(self.skip_reason) | ||
diff = og_connector_dir.diff(updated_connector_dir) | ||
exported_successfully = await diff.export(os.path.join(git.get_git_repo_path(), context.connector.code_directory)) | ||
if not exported_successfully: | ||
|
@@ -55,17 +68,57 @@ async def _run(self) -> StepResult: | |
exc_info=e, | ||
) | ||
|
||
def update_cdk_version(self, og_setup_py_content: str) -> str: | ||
async def upgrade_cdk_version_for_java_connector(self, og_connector_dir: Directory) -> Directory: | ||
if "build.gradle" not in await og_connector_dir.entries(): | ||
raise ValueError(f"Java connector {self.context.connector.technical_name} does not have a build.gradle file.") | ||
|
||
build_gradle = og_connector_dir.file("build.gradle") | ||
build_gradle_content = await build_gradle.contents() | ||
|
||
old_cdk_version_required = re.search(r"cdkVersionRequired *= *'(?P<version>[0-9]*\.[0-9]*\.[0-9]*)?'", build_gradle_content) | ||
# If there is no airbyte-cdk dependency, add the version | ||
if old_cdk_version_required is None: | ||
raise ValueError("Could not find airbyte-cdk dependency in build.gradle") | ||
|
||
if self.new_version == "latest": | ||
new_version = await cdk_helpers.get_latest_java_cdk_version(self.context.get_repo_dir()) | ||
else: | ||
new_version = self.new_version | ||
|
||
updated_build_gradle = build_gradle_content.replace(old_cdk_version_required.group("version"), new_version) | ||
|
||
use_local_cdk = re.search(r"useLocalCdk *=.*", updated_build_gradle) | ||
if use_local_cdk is not None: | ||
updated_build_gradle = updated_build_gradle.replace(use_local_cdk.group(), "useLocalCdk = false") | ||
|
||
return og_connector_dir.with_new_file("build.gradle", updated_build_gradle) | ||
|
||
async def upgrade_cdk_version_for_python_connector(self, og_connector_dir: Directory) -> Optional[Directory]: | ||
context = self.context | ||
og_connector_dir = await context.get_connector_dir() | ||
if "setup.py" not in await og_connector_dir.entries(): | ||
self.skip_reason = f"Python connector {self.context.connector.technical_name} does not have a setup.py file." | ||
return None | ||
setup_py = og_connector_dir.file("setup.py") | ||
setup_py_content = await setup_py.contents() | ||
|
||
airbyte_cdk_dependency = re.search( | ||
r"airbyte-cdk(?P<extra>\[[a-zA-Z0-9-]*\])?(?P<version>[<>=!~]+[0-9]*\.[0-9]*\.[0-9]*)?", og_setup_py_content | ||
r"airbyte-cdk(?P<extra>\[[a-zA-Z0-9-]*\])?(?P<version>[<>=!~]+[0-9]*(?:\.[0-9]*)?(?:\.[0-9]*)?)?", setup_py_content | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. the old regexp would not match |
||
) | ||
# If there is no airbyte-cdk dependency, add the version | ||
if airbyte_cdk_dependency is not None: | ||
new_version = f"airbyte-cdk{airbyte_cdk_dependency.group('extra') or ''}>={self.new_version}" | ||
return og_setup_py_content.replace(airbyte_cdk_dependency.group(), new_version) | ||
else: | ||
if airbyte_cdk_dependency is None: | ||
raise ValueError("Could not find airbyte-cdk dependency in setup.py") | ||
|
||
if self.new_version == "latest": | ||
new_version = cdk_helpers.get_latest_python_cdk_version() | ||
else: | ||
new_version = self.new_version | ||
|
||
new_version_str = f"airbyte-cdk{airbyte_cdk_dependency.group('extra') or ''}>={new_version}" | ||
updated_setup_py = setup_py_content.replace(airbyte_cdk_dependency.group(), new_version_str) | ||
|
||
return og_connector_dir.with_new_file("setup.py", updated_setup_py) | ||
|
||
|
||
async def run_connector_cdk_upgrade_pipeline( | ||
context: ConnectorContext, | ||
|
26 changes: 26 additions & 0 deletions
26
airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/cdk_helpers.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
# | ||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved. | ||
# | ||
import re | ||
|
||
import requests # type: ignore | ||
from dagger import Directory | ||
|
||
|
||
def get_latest_python_cdk_version() -> str: | ||
""" | ||
Get the latest version of airbyte-cdk from pypi | ||
""" | ||
cdk_pypi_url = "https://pypi.org/pypi/airbyte-cdk/json" | ||
response = requests.get(cdk_pypi_url) | ||
response.raise_for_status() | ||
package_info = response.json() | ||
return package_info["info"]["version"] | ||
|
||
|
||
async def get_latest_java_cdk_version(repo_dir: Directory) -> str: | ||
version_file_content = await repo_dir.file("airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties").contents() | ||
match = re.search(r"version *= *(?P<version>[0-9]*\.[0-9]*\.[0-9]*)", version_file_content) | ||
if match: | ||
return match.group("version") | ||
raise ValueError("Could not find version in version.properties") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
133 changes: 133 additions & 0 deletions
133
airbyte-ci/connectors/pipelines/tests/test_upgrade_java_cdk.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,133 @@ | ||
# | ||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved. | ||
# | ||
|
||
import json | ||
import random | ||
from pathlib import Path | ||
from typing import List | ||
from unittest.mock import AsyncMock, MagicMock | ||
|
||
import anyio | ||
import pytest | ||
from connector_ops.utils import Connector, ConnectorLanguage | ||
from dagger import Directory | ||
from pipelines.airbyte_ci.connectors.context import ConnectorContext | ||
from pipelines.airbyte_ci.connectors.publish import pipeline as publish_pipeline | ||
from pipelines.airbyte_ci.connectors.upgrade_cdk import pipeline as upgrade_cdk_pipeline | ||
from pipelines.models.steps import StepStatus | ||
|
||
pytestmark = [ | ||
pytest.mark.anyio, | ||
] | ||
|
||
|
||
@pytest.fixture | ||
def sample_connector(): | ||
return Connector("source-postgres") | ||
|
||
|
||
def get_sample_build_gradle(airbyte_cdk_version: str, useLocalCdk: str): | ||
return f"""import org.jsonschema2pojo.SourceType | ||
plugins {{ | ||
id 'application' | ||
id 'airbyte-java-connector' | ||
id "org.jsonschema2pojo" version "1.2.1" | ||
}} | ||
java {{ | ||
compileJava {{ | ||
options.compilerArgs += "-Xlint:-try,-rawtypes,-unchecked" | ||
}} | ||
}} | ||
airbyteJavaConnector {{ | ||
cdkVersionRequired = '{airbyte_cdk_version}' | ||
features = ['db-sources'] | ||
useLocalCdk = {useLocalCdk} | ||
}} | ||
application {{ | ||
mainClass = 'io.airbyte.integrations.source.postgres.PostgresSource' | ||
applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] | ||
}} | ||
""" | ||
|
||
|
||
@pytest.fixture | ||
def connector_context(sample_connector, dagger_client, current_platform): | ||
context = ConnectorContext( | ||
pipeline_name="test", | ||
connector=sample_connector, | ||
git_branch="test", | ||
git_revision="test", | ||
report_output_prefix="test", | ||
is_local=True, | ||
use_remote_secrets=True, | ||
targeted_platforms=[current_platform], | ||
) | ||
context.dagger_client = dagger_client | ||
return context | ||
|
||
|
||
@pytest.mark.parametrize( | ||
"build_gradle_content, expected_build_gradle_content", | ||
[ | ||
(get_sample_build_gradle("1.2.3", "false"), get_sample_build_gradle("6.6.6", "false")), | ||
(get_sample_build_gradle("1.2.3", "true"), get_sample_build_gradle("6.6.6", "false")), | ||
(get_sample_build_gradle("6.6.6", "false"), get_sample_build_gradle("6.6.6", "false")), | ||
(get_sample_build_gradle("6.6.6", "true"), get_sample_build_gradle("6.6.6", "false")), | ||
(get_sample_build_gradle("7.0.0", "false"), get_sample_build_gradle("6.6.6", "false")), | ||
(get_sample_build_gradle("7.0.0", "true"), get_sample_build_gradle("6.6.6", "false")), | ||
], | ||
) | ||
async def test_run_connector_cdk_upgrade_pipeline( | ||
connector_context: ConnectorContext, build_gradle_content: str, expected_build_gradle_content: str | ||
): | ||
full_og_connector_dir = await connector_context.get_connector_dir() | ||
updated_connector_dir = full_og_connector_dir.with_new_file("build.gradle", build_gradle_content) | ||
|
||
# For this test, replace the actual connector dir with an updated version that sets the build.gradle contents | ||
connector_context.get_connector_dir = AsyncMock(return_value=updated_connector_dir) | ||
|
||
# Mock the diff method to record the resulting directory and return a mock to not actually export the diff to the repo | ||
updated_connector_dir.diff = MagicMock(return_value=AsyncMock()) | ||
step = upgrade_cdk_pipeline.SetCDKVersion(connector_context, "6.6.6") | ||
step_result = await step.run() | ||
assert step_result.status == StepStatus.SUCCESS | ||
|
||
# Check that the resulting directory that got passed to the mocked diff method looks as expected | ||
resulting_directory: Directory = await full_og_connector_dir.diff(updated_connector_dir.diff.call_args[0][0]) | ||
files = await resulting_directory.entries() | ||
# validate only build.gradle is changed | ||
assert files == ["build.gradle"] | ||
build_gradle = resulting_directory.file("build.gradle") | ||
actual_build_gradle_content = await build_gradle.contents() | ||
assert expected_build_gradle_content == actual_build_gradle_content | ||
|
||
# Assert that the diff was exported to the repo | ||
assert updated_connector_dir.diff.return_value.export.call_count == 1 | ||
|
||
|
||
async def test_skip_connector_cdk_upgrade_pipeline_on_missing_build_gradle(connector_context: ConnectorContext): | ||
full_og_connector_dir = await connector_context.get_connector_dir() | ||
updated_connector_dir = full_og_connector_dir.without_file("build.gradle") | ||
|
||
connector_context.get_connector_dir = AsyncMock(return_value=updated_connector_dir) | ||
|
||
step = upgrade_cdk_pipeline.SetCDKVersion(connector_context, "6.6.6") | ||
step_result = await step.run() | ||
assert step_result.status == StepStatus.FAILURE | ||
|
||
|
||
async def test_fail_connector_cdk_upgrade_pipeline_on_missing_airbyte_cdk(connector_context: ConnectorContext): | ||
full_og_connector_dir = await connector_context.get_connector_dir() | ||
updated_connector_dir = full_og_connector_dir.with_new_file("build.gradle", get_sample_build_gradle("abc", "false")) | ||
|
||
connector_context.get_connector_dir = AsyncMock(return_value=updated_connector_dir) | ||
|
||
step = upgrade_cdk_pipeline.SetCDKVersion(connector_context, "6.6.6") | ||
step_result = await step.run() | ||
assert step_result.status == StepStatus.FAILURE |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
now, "latest" has different meaning for python vs java CDK. Therefore, I'm just passing the
"latest"
string and will resolve it later