Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

migrate prefect-gitlab to pydantic v2 #13656

Merged
merged 3 commits into from
May 30, 2024
Merged

migrate prefect-gitlab to pydantic v2 #13656

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9fff244
prefect-gitlab
aaazzam May 30, 2024
be2122d
Update repositories.py
aaazzam May 30, 2024
ef722c5
Update repositories.py
aaazzam May 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 7 additions & 13 deletions src/integrations/prefect-gitlab/prefect_gitlab/credentials.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,12 @@
"""Module used to enable authenticated interactions with GitLab"""

from typing import Optional

from gitlab import Gitlab
from pydantic import VERSION as PYDANTIC_VERSION
from pydantic import Field, SecretStr

from prefect.blocks.core import Block

if PYDANTIC_VERSION.startswith("2."):
from pydantic.v1 import Field, HttpUrl, SecretStr
else:
from pydantic import Field, HttpUrl, SecretStr


class GitLabCredentials(Block):
"""
Expand All @@ -29,17 +26,14 @@ class GitLabCredentials(Block):
"""

_block_type_name = "GitLab Credentials"
_logo_url = HttpUrl(
url="https://images.ctfassets.net/gm98wzqotmnx/55edIimT4g9gbjhkh5a3Sp/dfdb9391d8f45c2e93e72e3a4d350771/gitlab-logo-500.png?h=250", # noqa
scheme="https",
)
_logo_url = "https://images.ctfassets.net/gm98wzqotmnx/55edIimT4g9gbjhkh5a3Sp/dfdb9391d8f45c2e93e72e3a4d350771/gitlab-logo-500.png?h=250"

token: SecretStr = Field(
name="Personal Access Token",
token: Optional[SecretStr] = Field(
title="Personal Access Token",
default=None,
description="A GitLab Personal Access Token with read_repository scope.",
)
url: str = Field(
url: Optional[str] = Field(
default=None, title="URL", description="URL to self-hosted GitLab instances."
)

Expand Down
42 changes: 5 additions & 37 deletions src/integrations/prefect-gitlab/prefect_gitlab/repositories.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,26 +40,20 @@
private_gitlab_block.save()
```
"""

import io
import shutil
import urllib.parse
from distutils.dir_util import copy_tree
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Optional, Tuple, Union

from pydantic import VERSION as PYDANTIC_VERSION
from pydantic import Field
from tenacity import retry, stop_after_attempt, wait_fixed, wait_random

from prefect.exceptions import InvalidRepositoryURLError
from prefect.filesystems import ReadableDeploymentStorage
from prefect.utilities.asyncutils import sync_compatible
from prefect.utilities.processutils import run_process

if PYDANTIC_VERSION.startswith("2."):
from pydantic.v1 import Field, HttpUrl, validator
else:
from pydantic import Field, HttpUrl, validator

from prefect_gitlab.credentials import GitLabCredentials

# Create get_directory retry settings
Expand All @@ -79,10 +73,7 @@ class GitLabRepository(ReadableDeploymentStorage):
"""

_block_type_name = "GitLab Repository"
_logo_url = HttpUrl(
url="https://images.ctfassets.net/gm98wzqotmnx/55edIimT4g9gbjhkh5a3Sp/dfdb9391d8f45c2e93e72e3a4d350771/gitlab-logo-500.png?h=250", # noqa
scheme="https",
)
_logo_url = "https://images.ctfassets.net/gm98wzqotmnx/55edIimT4g9gbjhkh5a3Sp/dfdb9391d8f45c2e93e72e3a4d350771/gitlab-logo-500.png?h=250"
_description = "Interact with files stored in GitLab repositories."

repository: str = Field(
Expand All @@ -107,29 +98,6 @@ class GitLabRepository(ReadableDeploymentStorage):
"private GitLab repos.",
)

@validator("credentials")
def _ensure_credentials_go_with_http(cls, v: str, values: dict) -> str:
"""Ensure that credentials are not provided with 'SSH' formatted GitLub URLs.
Note: validates `access_token` specifically so that it only fires when
private repositories are used.
"""
if v is not None:
if urllib.parse.urlparse(values["repository"]).scheme not in [
"https",
"http",
]:
raise InvalidRepositoryURLError(
(
"Credentials can only be used with GitLab repositories "
"using the 'HTTPS'/'HTTP' format. You must either remove the "
"credential if you wish to use the 'SSH' format and are not "
"using a private repository, or you must change the repository "
"URL to the 'HTTPS'/'HTTP' format."
)
)

return v

def _create_repo_url(self) -> str:
"""Format the URL provided to the `git clone` command.
For private repos: https://<oauth-key>@gitlab.com/<username>/<repo>.git
Expand Down Expand Up @@ -213,4 +181,4 @@ async def get_directory(
dst_dir=local_path, src_dir=tmp_dir, sub_directory=from_path
)

copy_tree(src=content_source, dst=content_destination)
shutil.copytree(src=content_source, dst=content_destination)
39 changes: 4 additions & 35 deletions src/integrations/prefect-gitlab/tests/test_repositories.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,13 @@
from tempfile import TemporaryDirectory
from typing import Set, Tuple

import pytest
from pydantic import VERSION as PYDANTIC_VERSION

from prefect.exceptions import InvalidRepositoryURLError
from prefect.testing.utilities import AsyncMock

if PYDANTIC_VERSION.startswith("2."):
from pydantic.v1 import SecretStr
else:
from pydantic import SecretStr

import prefect_gitlab
import pytest
from prefect_gitlab.credentials import GitLabCredentials
from prefect_gitlab.repositories import GitLabRepository # noqa: E402
from pydantic import SecretStr

from prefect.testing.utilities import AsyncMock


class TestGitLab:
Expand Down Expand Up @@ -170,30 +163,6 @@ class p:
]
assert mock.await_args[0][0][: len(expected_cmd)] == expected_cmd

async def test_ssh_fails_with_credential(self, monkeypatch):
"""Ensure that credentials cannot be passed in if the URL is not in the HTTPS/HTTP
format.
"""

class p:
returncode = 0

mock = AsyncMock(return_value=p())
monkeypatch.setattr(prefect_gitlab.repositories, "run_process", mock)
credential = "XYZ"
error_msg = (
"Credentials can only be used with GitLab repositories "
"using the 'HTTPS'/'HTTP' format. You must either remove the "
"credential if you wish to use the 'SSH' format and are not "
"using a private repository, or you must change the repository "
"URL to the 'HTTPS'/'HTTP' format."
)
with pytest.raises(InvalidRepositoryURLError, match=error_msg):
GitLabRepository(
repository="git@gitlab.com:PrefectHQ/prefect.git",
credentials=GitLabCredentials(token=SecretStr(credential)),
)

async def test_dir_contents_copied_correctly_with_get_directory(self, monkeypatch): # noqa
"""Check that `get_directory` is able to correctly copy contents from src->dst""" # noqa

Expand Down
Loading