From c96a6853d30b5bdb934774145838bbbbd69a08a1 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Thu, 26 Sep 2024 16:29:54 -0700 Subject: [PATCH 1/6] pass and init proxy settings in dulwich and pygit2 --- src/scmrepo/git/backend/dulwich/__init__.py | 6 ++++-- src/scmrepo/git/backend/pygit2/__init__.py | 6 ++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/scmrepo/git/backend/dulwich/__init__.py b/src/scmrepo/git/backend/dulwich/__init__.py index c03d6b60..a4280168 100644 --- a/src/scmrepo/git/backend/dulwich/__init__.py +++ b/src/scmrepo/git/backend/dulwich/__init__.py @@ -16,6 +16,7 @@ Union, ) +from dulwich.config import ConfigFile, StackedConfig from funcy import cached_property, reraise from scmrepo.exceptions import AuthError, CloneError, InvalidRemote, RevError, SCMError @@ -27,7 +28,7 @@ if TYPE_CHECKING: from dulwich.client import SSHVendor - from dulwich.config import ConfigFile, StackedConfig + from dulwich.config import ConfigFile from dulwich.repo import Repo from scmrepo.git.objects import GitCommit @@ -579,7 +580,8 @@ def iter_remote_refs(self, url: str, base: Optional[str] = None, **kwargs): try: _remote, location = get_remote_repo(self.repo, url) - client, path = get_transport_and_path(location, **kwargs) + _config = kwargs.pop("config", StackedConfig.default()) + client, path = get_transport_and_path(location, config=_config, **kwargs) except Exception as exc: raise InvalidRemote(url) from exc diff --git a/src/scmrepo/git/backend/pygit2/__init__.py b/src/scmrepo/git/backend/pygit2/__init__.py index a3a3faea..61cb9364 100644 --- a/src/scmrepo/git/backend/pygit2/__init__.py +++ b/src/scmrepo/git/backend/pygit2/__init__.py @@ -701,15 +701,13 @@ def _default_status( remote_refs: dict[str, Oid] = ( { head["name"]: head["oid"] - for head in remote.ls_remotes(callbacks=cb) + for head in remote.ls_remotes(callbacks=cb, proxy=True) } if not force else {} ) remote.fetch( - refspecs=refspecs, - callbacks=cb, - message="fetch", + refspecs=refspecs, callbacks=cb, message="fetch", proxy=True ) result: dict[str, SyncStatus] = {} From 52d2e9b03ce8d240225f18c2a2cc11c9a2f1b89f Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Sat, 28 Sep 2024 18:43:00 -0700 Subject: [PATCH 2/6] add tests, fix more methods in dulwich --- .gitignore | 1 + pyproject.toml | 5 +- src/scmrepo/git/backend/dulwich/__init__.py | 7 +- tests/test_git.py | 90 ++++++++++++++++++++- 4 files changed, 98 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 674cab40..3ce84936 100644 --- a/.gitignore +++ b/.gitignore @@ -138,3 +138,4 @@ dmypy.json cython_debug/ .DS_Store +.vscode/ diff --git a/pyproject.toml b/pyproject.toml index d9f3b767..549c2ad9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,8 @@ tests = [ "pytest-mock", "pytest-sugar", "pytest-test-utils>=0.1.0,<0.2", + "proxy.py", + "pytest-socket", ] dev = [ "mypy==1.11.2", @@ -66,13 +68,14 @@ where = ["src"] namespaces = false [tool.pytest.ini_options] -addopts = "-ra" +addopts = "-ra --allow-unix-socket" markers = [ "skip_git_backend: skip tests for given backend", "slow: mark test as slow to run", ] asyncio_mode = "auto" + [tool.coverage.run] branch = true source = ["scmrepo", "tests"] diff --git a/src/scmrepo/git/backend/dulwich/__init__.py b/src/scmrepo/git/backend/dulwich/__init__.py index a4280168..c54b0678 100644 --- a/src/scmrepo/git/backend/dulwich/__init__.py +++ b/src/scmrepo/git/backend/dulwich/__init__.py @@ -618,7 +618,8 @@ def push_refspecs( # noqa: C901 try: _remote, location = get_remote_repo(self.repo, url) - client, path = get_transport_and_path(location, **kwargs) + _config = kwargs.pop("config", StackedConfig.default()) + client, path = get_transport_and_path(location, config=_config, **kwargs) except Exception as exc: raise SCMError(f"'{url}' is not a valid Git remote or URL") from exc @@ -725,7 +726,8 @@ def determine_wants( with reraise(Exception, SCMError(f"'{url}' is not a valid Git remote or URL")): _remote, location = get_remote_repo(self.repo, url) - client, path = get_transport_and_path(location, **kwargs) + _config = kwargs.pop("config", StackedConfig.default()) + client, path = get_transport_and_path(location, config=_config, **kwargs) with reraise( (NotGitRepository, KeyError), @@ -911,6 +913,7 @@ def validate_git_remote(self, url: str, **kwargs): try: _, location = get_remote_repo(self.repo, url) + _config = kwargs.pop("config", StackedConfig.default()) client, path = get_transport_and_path(location, **kwargs) except Exception as exc: raise InvalidRemote(url) from exc diff --git a/tests/test_git.py b/tests/test_git.py index 6e394c76..069d6204 100644 --- a/tests/test_git.py +++ b/tests/test_git.py @@ -1,5 +1,6 @@ import os import shutil +from pathlib import Path from typing import Any, Optional import pytest @@ -7,14 +8,20 @@ from asyncssh.connection import SSHClientConnection from dulwich.client import LocalGitClient from git import Repo as GitPythonRepo +from proxy import TestCase as ProxyTestCase from pygit2 import GitError from pygit2.remotes import Remote from pytest_mock import MockerFixture from pytest_test_utils import TempDirFactory, TmpDir from pytest_test_utils.matchers import Matcher -from scmrepo.exceptions import InvalidRemote, MergeConflictError, RevError, SCMError -from scmrepo.git import Git +from scmrepo.exceptions import ( + InvalidRemote, + MergeConflictError, + RevError, + SCMError, +) +from scmrepo.git import Git, GitBackends from scmrepo.git.objects import GitTag from .conftest import backends @@ -22,6 +29,13 @@ # pylint: disable=redefined-outer-name,unused-argument,protected-access +BAD_PROXY_CONFIG = """[http] +proxy = "http://bad-proxy.dvc.org" +[https] +proxy = "http://bad-proxy.dvc.org" +""" + + @pytest.fixture def submodule_dir(tmp_dir: TmpDir, scm: Git): scm.commit("init") @@ -941,6 +955,78 @@ def test_clone( assert fobj.read().strip() == "foo" +@pytest.fixture +def proxy_server(): + class _ProxyServer(ProxyTestCase): + pass + + _ProxyServer.setUpClass() + yield f"http://{_ProxyServer.PROXY.flags.hostname}:{_ProxyServer.PROXY.flags.port}" + _ProxyServer.tearDownClass() + + +@pytest.mark.allow_hosts(["127.0.0.1", "::1"]) +def test_clone_proxy_server(proxy_server: str, scm: Git, git: Git, tmp_dir: TmpDir): + url = "https://github.com/iterative/dvcyaml-schema" + + p = Path(os.environ["HOME"]) / ".gitconfig" + p.write_text(BAD_PROXY_CONFIG) + with pytest.raises(Exception): # noqa: PT011, B017 + git.clone(url, "dir") + + mock_config_content = f"""[http]\n +proxy = {proxy_server} +[https] +proxy = {proxy_server} +""" + + p.write_text(mock_config_content) + git.clone(url, "dir") + + +@pytest.mark.allow_hosts(["127.0.0.1", "::1"]) +def test_iter_remote_refs_proxy_server(proxy_server: str, scm: Git, tmp_dir: TmpDir): + url = "https://github.com/iterative/dvcyaml-schema" + git = GitBackends.DEFAULT.get("dulwich")(".") + + p = Path(os.environ["HOME"]) / ".gitconfig" + p.write_text(BAD_PROXY_CONFIG) + with pytest.raises(Exception): # noqa: PT011, B017 + list(git.iter_remote_refs(url)) + + mock_config_content = f"""[http] +proxy = {proxy_server} +[https] +proxy = {proxy_server} +""" + + p.write_text(mock_config_content) + res = list(git.iter_remote_refs(url)) + assert res + + +@pytest.mark.skip_git_backend("gitpython") +@pytest.mark.allow_hosts(["127.0.0.1", "::1"]) +def test_fetch_refspecs_proxy_server( + proxy_server: str, scm: Git, git: Git, tmp_dir: TmpDir +): + url = "https://github.com/iterative/dvcyaml-schema" + + p = Path(os.environ["HOME"]) / ".gitconfig" + p.write_text(BAD_PROXY_CONFIG) + with pytest.raises(Exception): # noqa: PT011, B017 + git.fetch_refspecs(url, ["refs/heads/master:refs/heads/master"]) + + mock_config_content = f"""[http] +proxy = {proxy_server} +[https] +proxy = {proxy_server} +""" + + p.write_text(mock_config_content) + git.fetch_refspecs(url, "refs/heads/master:refs/heads/master") + + @pytest.mark.skip_git_backend("pygit2") def test_fetch(tmp_dir: TmpDir, scm: Git, git: Git, tmp_dir_factory: TempDirFactory): tmp_dir.gen("foo", "foo") From 42cc27aa58474ba9a37a160957f867a2027e1b4f Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Sat, 28 Sep 2024 18:52:03 -0700 Subject: [PATCH 3/6] lint --- tests/test_git.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_git.py b/tests/test_git.py index 069d6204..8d78e941 100644 --- a/tests/test_git.py +++ b/tests/test_git.py @@ -987,7 +987,7 @@ def test_clone_proxy_server(proxy_server: str, scm: Git, git: Git, tmp_dir: TmpD @pytest.mark.allow_hosts(["127.0.0.1", "::1"]) def test_iter_remote_refs_proxy_server(proxy_server: str, scm: Git, tmp_dir: TmpDir): url = "https://github.com/iterative/dvcyaml-schema" - git = GitBackends.DEFAULT.get("dulwich")(".") + git = GitBackends.DEFAULT["dulwich"](".") p = Path(os.environ["HOME"]) / ".gitconfig" p.write_text(BAD_PROXY_CONFIG) From 5ff170db94fa004d8ece2a9065db84a4d38437d9 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Sat, 28 Sep 2024 19:06:47 -0700 Subject: [PATCH 4/6] debug tests --- tests/test_git.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_git.py b/tests/test_git.py index 8d78e941..d8bfe376 100644 --- a/tests/test_git.py +++ b/tests/test_git.py @@ -989,7 +989,12 @@ def test_iter_remote_refs_proxy_server(proxy_server: str, scm: Git, tmp_dir: Tmp url = "https://github.com/iterative/dvcyaml-schema" git = GitBackends.DEFAULT["dulwich"](".") - p = Path(os.environ["HOME"]) / ".gitconfig" + import sys + + if sys.platform == "win32": + p = Path(os.environ["USERPROFILE"]) / ".gitconfig" + else: + p = Path(os.environ["HOME"]) / ".gitconfig" p.write_text(BAD_PROXY_CONFIG) with pytest.raises(Exception): # noqa: PT011, B017 list(git.iter_remote_refs(url)) @@ -1006,7 +1011,6 @@ def test_iter_remote_refs_proxy_server(proxy_server: str, scm: Git, tmp_dir: Tmp @pytest.mark.skip_git_backend("gitpython") -@pytest.mark.allow_hosts(["127.0.0.1", "::1"]) def test_fetch_refspecs_proxy_server( proxy_server: str, scm: Git, git: Git, tmp_dir: TmpDir ): From 47c93b5b2d769e5d7b7acbf035bc227cf569c45a Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Sat, 28 Sep 2024 19:14:37 -0700 Subject: [PATCH 5/6] drop pytest socket --- pyproject.toml | 3 +-- tests/test_git.py | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 549c2ad9..b6fb9b61 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,6 @@ tests = [ "pytest-sugar", "pytest-test-utils>=0.1.0,<0.2", "proxy.py", - "pytest-socket", ] dev = [ "mypy==1.11.2", @@ -68,7 +67,7 @@ where = ["src"] namespaces = false [tool.pytest.ini_options] -addopts = "-ra --allow-unix-socket" +addopts = "-ra" markers = [ "skip_git_backend: skip tests for given backend", "slow: mark test as slow to run", diff --git a/tests/test_git.py b/tests/test_git.py index d8bfe376..cc66eac0 100644 --- a/tests/test_git.py +++ b/tests/test_git.py @@ -965,7 +965,6 @@ class _ProxyServer(ProxyTestCase): _ProxyServer.tearDownClass() -@pytest.mark.allow_hosts(["127.0.0.1", "::1"]) def test_clone_proxy_server(proxy_server: str, scm: Git, git: Git, tmp_dir: TmpDir): url = "https://github.com/iterative/dvcyaml-schema" @@ -984,7 +983,6 @@ def test_clone_proxy_server(proxy_server: str, scm: Git, git: Git, tmp_dir: TmpD git.clone(url, "dir") -@pytest.mark.allow_hosts(["127.0.0.1", "::1"]) def test_iter_remote_refs_proxy_server(proxy_server: str, scm: Git, tmp_dir: TmpDir): url = "https://github.com/iterative/dvcyaml-schema" git = GitBackends.DEFAULT["dulwich"](".") From 9b48130f26dac258b4fbd203d18d330cbee9ae37 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Sat, 28 Sep 2024 19:40:34 -0700 Subject: [PATCH 6/6] fix windows path to gitconfig --- tests/test_git.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/test_git.py b/tests/test_git.py index cc66eac0..268fbd1c 100644 --- a/tests/test_git.py +++ b/tests/test_git.py @@ -968,7 +968,10 @@ class _ProxyServer(ProxyTestCase): def test_clone_proxy_server(proxy_server: str, scm: Git, git: Git, tmp_dir: TmpDir): url = "https://github.com/iterative/dvcyaml-schema" - p = Path(os.environ["HOME"]) / ".gitconfig" + p = ( + Path(os.environ["HOME"] if "HOME" in os.environ else os.environ["USERPROFILE"]) + / ".gitconfig" + ) p.write_text(BAD_PROXY_CONFIG) with pytest.raises(Exception): # noqa: PT011, B017 git.clone(url, "dir") @@ -987,12 +990,10 @@ def test_iter_remote_refs_proxy_server(proxy_server: str, scm: Git, tmp_dir: Tmp url = "https://github.com/iterative/dvcyaml-schema" git = GitBackends.DEFAULT["dulwich"](".") - import sys - - if sys.platform == "win32": - p = Path(os.environ["USERPROFILE"]) / ".gitconfig" - else: - p = Path(os.environ["HOME"]) / ".gitconfig" + p = ( + Path(os.environ["HOME"] if "HOME" in os.environ else os.environ["USERPROFILE"]) + / ".gitconfig" + ) p.write_text(BAD_PROXY_CONFIG) with pytest.raises(Exception): # noqa: PT011, B017 list(git.iter_remote_refs(url)) @@ -1014,7 +1015,10 @@ def test_fetch_refspecs_proxy_server( ): url = "https://github.com/iterative/dvcyaml-schema" - p = Path(os.environ["HOME"]) / ".gitconfig" + p = ( + Path(os.environ["HOME"] if "HOME" in os.environ else os.environ["USERPROFILE"]) + / ".gitconfig" + ) p.write_text(BAD_PROXY_CONFIG) with pytest.raises(Exception): # noqa: PT011, B017 git.fetch_refspecs(url, ["refs/heads/master:refs/heads/master"])