From 983898a683877ac6c60e005c9c8759f84551d2b7 Mon Sep 17 00:00:00 2001 From: Dmitry Meyer Date: Thu, 18 Sep 2025 14:58:26 +0000 Subject: [PATCH 1/4] Ignore Git configs when checking repo creds Fixes: https://github.com/dstackai/dstack/issues/3115 --- src/dstack/_internal/core/services/repos.py | 9 +++++---- src/dstack/_internal/utils/ssh.py | 20 ++++++++++++++++++-- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/dstack/_internal/core/services/repos.py b/src/dstack/_internal/core/services/repos.py index b81fd4e1e..edf99375e 100644 --- a/src/dstack/_internal/core/services/repos.py +++ b/src/dstack/_internal/core/services/repos.py @@ -87,8 +87,9 @@ def _get_repo_creds_and_default_branch_ssh( url: GitRepoURL, identity_file: PathLike, private_key: str ) -> tuple[RemoteRepoCreds, Optional[str]]: _url = url.as_ssh() + env = make_git_env(disable_config=True, identity_file=identity_file) try: - default_branch = _get_repo_default_branch(_url, make_git_env(identity_file=identity_file)) + default_branch = _get_repo_default_branch(_url, env) except GitCommandError as e: message = f"Cannot access `{_url}` using the `{identity_file}` private SSH key" raise InvalidRepoCredentialsError(message) from e @@ -104,8 +105,9 @@ def _get_repo_creds_and_default_branch_https( url: GitRepoURL, oauth_token: Optional[str] = None ) -> tuple[RemoteRepoCreds, Optional[str]]: _url = url.as_https() + env = make_git_env(disable_config=True) try: - default_branch = _get_repo_default_branch(url.as_https(oauth_token), make_git_env()) + default_branch = _get_repo_default_branch(url.as_https(oauth_token), env) except GitCommandError as e: message = f"Cannot access `{_url}`" if oauth_token is not None: @@ -122,8 +124,7 @@ def _get_repo_creds_and_default_branch_https( def _get_repo_default_branch(url: str, env: dict[str, str]) -> Optional[str]: # output example: "ref: refs/heads/dev\tHEAD\n545344f77c0df78367085952a97fc3a058eb4c65\tHEAD" - # Disable credential helpers to exclude any default credentials from being used - output: str = git.cmd.Git()(c="credential.helper=").ls_remote("--symref", url, "HEAD", env=env) + output: str = git.cmd.Git().ls_remote("--symref", url, "HEAD", env=env) for line in output.splitlines(): # line format: ` TAB LF` oid, _, ref = line.partition("\t") diff --git a/src/dstack/_internal/utils/ssh.py b/src/dstack/_internal/utils/ssh.py index f0dafc7f7..cbf7d78c0 100644 --- a/src/dstack/_internal/utils/ssh.py +++ b/src/dstack/_internal/utils/ssh.py @@ -50,8 +50,24 @@ def make_ssh_command_for_git(identity_file: PathLike) -> str: ) -def make_git_env(*, identity_file: Optional[PathLike] = None) -> dict[str, str]: - env: dict[str, str] = {"GIT_TERMINAL_PROMPT": "0"} +def make_git_env( + *, + disable_prompt: bool = True, + disable_config: bool = False, + identity_file: Optional[PathLike] = None, +) -> dict[str, str]: + env: dict[str, str] = {} + if disable_prompt: + # Fail with error instead of prompting on the terminal (e.g., when asking for + # HTTP authentication) + env["GIT_TERMINAL_PROMPT"] = "0" + if disable_config: + # Disable system-wide config (usually /etc/gitconfig) + env["GIT_CONFIG_SYSTEM"] = os.devnull + # Disable user (aka "global") config ($XDG_CONFIG_HOME/git/config or ~/.git/config) + env["GIT_CONFIG_GLOBAL"] = os.devnull + # Disable repo (aka "local") config (./.git/config) + env["GIT_DIR"] = os.devnull if identity_file is not None: env["GIT_SSH_COMMAND"] = make_ssh_command_for_git(identity_file) return env From fa076310cb2e06327381c35758ed0c4ecf9e626e Mon Sep 17 00:00:00 2001 From: Dmitry Meyer Date: Fri, 19 Sep 2025 10:52:05 +0000 Subject: [PATCH 2/4] Restore `-c credential.helper=` option Required for Git shipped with XCode --- src/dstack/_internal/core/services/repos.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/dstack/_internal/core/services/repos.py b/src/dstack/_internal/core/services/repos.py index edf99375e..69e4a382a 100644 --- a/src/dstack/_internal/core/services/repos.py +++ b/src/dstack/_internal/core/services/repos.py @@ -123,8 +123,20 @@ def _get_repo_creds_and_default_branch_https( def _get_repo_default_branch(url: str, env: dict[str, str]) -> Optional[str]: + # Git shipped by Apple with XCode is patched to support an additional config scope + # above "system" called "xcode". There is no option in `git config list` to show this config, + # but you can list the merged config (`git config list` without options) and then exclude + # all settings listed in `git config list --{system,global,local,worktree}`. + # As of time of writing, there are only two settings in the "xcode" config, one of which breaks + # our "is repo public?" check, namely "credential.helper=osxkeychain". + # As there is no way to disable "xcode" config (no env variable, no CLI option, etc.), + # the only way to disable credential helper is to override this specific setting with an empty + # string via command line argument: `git -c credential.helper= COMMAND [ARGS ...]`. + # See: https://github.com/git/git/commit/3d4355712b9fe77a96ad4ad877d92dc7ff6e0874 + # See: https://gist.github.com/ChrisTollefson/ab9c0a5d1dd4dd615217345c6936a307 + _git = git.cmd.Git()(c="credential.helper=") # output example: "ref: refs/heads/dev\tHEAD\n545344f77c0df78367085952a97fc3a058eb4c65\tHEAD" - output: str = git.cmd.Git().ls_remote("--symref", url, "HEAD", env=env) + output: str = _git.ls_remote("--symref", url, "HEAD", env=env) for line in output.splitlines(): # line format: ` TAB LF` oid, _, ref = line.partition("\t") From a8a0c0e86c5b6da69e30df4b746c5da7bb887974 Mon Sep 17 00:00:00 2001 From: peterschmidt85 Date: Fri, 19 Sep 2025 14:07:41 +0200 Subject: [PATCH 3/4] [Bug]: dstack misconfigures Git credentials for private repos #3115 Added debug logs --- src/dstack/_internal/core/services/repos.py | 80 +++++++++++++++++++-- 1 file changed, 73 insertions(+), 7 deletions(-) diff --git a/src/dstack/_internal/core/services/repos.py b/src/dstack/_internal/core/services/repos.py index 69e4a382a..13d78ab55 100644 --- a/src/dstack/_internal/core/services/repos.py +++ b/src/dstack/_internal/core/services/repos.py @@ -36,24 +36,59 @@ def get_repo_creds_and_default_branch( # no auth with suppress(InvalidRepoCredentialsError): - return _get_repo_creds_and_default_branch_https(url) + creds, default_branch = _get_repo_creds_and_default_branch_https(url) + logger.debug( + "Git repo %s is public. Using no auth. Default branch: %s", repo_url, default_branch + ) + return creds, default_branch # ssh key provided by the user or pulled from the server if identity_file is not None or private_key is not None: if identity_file is not None: private_key = _read_private_key(identity_file) - return _get_repo_creds_and_default_branch_ssh(url, identity_file, private_key) + creds, default_branch = _get_repo_creds_and_default_branch_ssh( + url, identity_file, private_key + ) + logger.debug( + "Git repo %s is private. Using identity file: %s. Default branch: %s", + repo_url, + identity_file, + default_branch, + ) + return creds, default_branch elif private_key is not None: with NamedTemporaryFile("w+", 0o600) as f: f.write(private_key) f.flush() - return _get_repo_creds_and_default_branch_ssh(url, f.name, private_key) + creds, default_branch = _get_repo_creds_and_default_branch_ssh( + url, f.name, private_key + ) + masked_key = "***" + private_key[-10:] if len(private_key) > 10 else "***MASKED***" + logger.debug( + "Git repo %s is private. Using private key: %s. Default branch: %s", + repo_url, + masked_key, + default_branch, + ) + return creds, default_branch else: assert False, "should not reach here" # oauth token provided by the user or pulled from the server if oauth_token is not None: - return _get_repo_creds_and_default_branch_https(url, oauth_token) + creds, default_branch = _get_repo_creds_and_default_branch_https(url, oauth_token) + masked_token = ( + len(oauth_token[:-4]) * "*" + oauth_token[-4:] + if len(oauth_token) > 4 + else "***MASKED***" + ) + logger.debug( + "Git repo %s is private. Using provided OAuth token: %s. Default branch: %s", + repo_url, + masked_token, + default_branch, + ) + return creds, default_branch # key from ssh config identities = get_host_config(url.original_host).get("identityfile") @@ -61,7 +96,16 @@ def get_repo_creds_and_default_branch( _identity_file = identities[0] with suppress(InvalidRepoCredentialsError): _private_key = _read_private_key(_identity_file) - return _get_repo_creds_and_default_branch_ssh(url, _identity_file, _private_key) + creds, default_branch = _get_repo_creds_and_default_branch_ssh( + url, _identity_file, _private_key + ) + logger.debug( + "Git repo %s is private. Using SSH config identity file: %s. Default branch: %s", + repo_url, + _identity_file, + default_branch, + ) + return creds, default_branch # token from gh config if os.path.exists(gh_config_path): @@ -70,13 +114,35 @@ def get_repo_creds_and_default_branch( _oauth_token = gh_hosts.get(url.host, {}).get("oauth_token") if _oauth_token is not None: with suppress(InvalidRepoCredentialsError): - return _get_repo_creds_and_default_branch_https(url, _oauth_token) + creds, default_branch = _get_repo_creds_and_default_branch_https(url, _oauth_token) + masked_token = ( + len(_oauth_token[:-4]) * "*" + _oauth_token[-4:] + if len(_oauth_token) > 4 + else "***MASKED***" + ) + logger.debug( + "Git repo %s is private. Using GitHub config token: %s from %s. Default branch: %s", + repo_url, + masked_token, + gh_config_path, + default_branch, + ) + return creds, default_branch # default user key if os.path.exists(default_ssh_key): with suppress(InvalidRepoCredentialsError): _private_key = _read_private_key(default_ssh_key) - return _get_repo_creds_and_default_branch_ssh(url, default_ssh_key, _private_key) + creds, default_branch = _get_repo_creds_and_default_branch_ssh( + url, default_ssh_key, _private_key + ) + logger.debug( + "Git repo %s is private. Using default identity file: %s. Default branch: %s", + repo_url, + default_ssh_key, + default_branch, + ) + return creds, default_branch raise InvalidRepoCredentialsError( "No valid default Git credentials found. Pass valid `--token` or `--git-identity`." From 229a597cd8225f1dd15457bd358bbb1a41b6df1a Mon Sep 17 00:00:00 2001 From: Dmitry Meyer Date: Fri, 19 Sep 2025 12:20:00 +0000 Subject: [PATCH 4/4] Disable askpass --- src/dstack/_internal/core/services/repos.py | 15 +++++++++++++-- src/dstack/_internal/utils/ssh.py | 4 ++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/dstack/_internal/core/services/repos.py b/src/dstack/_internal/core/services/repos.py index 13d78ab55..f68a0d0d5 100644 --- a/src/dstack/_internal/core/services/repos.py +++ b/src/dstack/_internal/core/services/repos.py @@ -153,7 +153,7 @@ def _get_repo_creds_and_default_branch_ssh( url: GitRepoURL, identity_file: PathLike, private_key: str ) -> tuple[RemoteRepoCreds, Optional[str]]: _url = url.as_ssh() - env = make_git_env(disable_config=True, identity_file=identity_file) + env = _make_git_env_for_creds_check(identity_file=identity_file) try: default_branch = _get_repo_default_branch(_url, env) except GitCommandError as e: @@ -171,7 +171,7 @@ def _get_repo_creds_and_default_branch_https( url: GitRepoURL, oauth_token: Optional[str] = None ) -> tuple[RemoteRepoCreds, Optional[str]]: _url = url.as_https() - env = make_git_env(disable_config=True) + env = _make_git_env_for_creds_check() try: default_branch = _get_repo_default_branch(url.as_https(oauth_token), env) except GitCommandError as e: @@ -188,6 +188,17 @@ def _get_repo_creds_and_default_branch_https( return creds, default_branch +def _make_git_env_for_creds_check(identity_file: Optional[PathLike] = None) -> dict[str, str]: + # Our goal is to check if _provided_ creds (if any) are correct, so we need to be sure that + # only the provided creds are used, without falling back to any additional mechanisms. + # To do this, we: + # 1. Disable all configs to ignore any stored creds + # 2. Disable askpass to avoid asking for creds interactively or fetching stored creds from + # a non-interactive askpass helper (for example, VS Code sets GIT_ASKPASS to its own helper, + # which silently provides creds to Git). + return make_git_env(disable_config=True, disable_askpass=True, identity_file=identity_file) + + def _get_repo_default_branch(url: str, env: dict[str, str]) -> Optional[str]: # Git shipped by Apple with XCode is patched to support an additional config scope # above "system" called "xcode". There is no option in `git config list` to show this config, diff --git a/src/dstack/_internal/utils/ssh.py b/src/dstack/_internal/utils/ssh.py index cbf7d78c0..851103af6 100644 --- a/src/dstack/_internal/utils/ssh.py +++ b/src/dstack/_internal/utils/ssh.py @@ -53,6 +53,7 @@ def make_ssh_command_for_git(identity_file: PathLike) -> str: def make_git_env( *, disable_prompt: bool = True, + disable_askpass: bool = False, disable_config: bool = False, identity_file: Optional[PathLike] = None, ) -> dict[str, str]: @@ -61,6 +62,9 @@ def make_git_env( # Fail with error instead of prompting on the terminal (e.g., when asking for # HTTP authentication) env["GIT_TERMINAL_PROMPT"] = "0" + if disable_askpass: + env["GIT_ASKPASS"] = "" + env["SSH_ASKPASS"] = "" if disable_config: # Disable system-wide config (usually /etc/gitconfig) env["GIT_CONFIG_SYSTEM"] = os.devnull