Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[core] Workaround erroneous urllib Windows proxy parsing #7092

Merged
merged 3 commits into from May 27, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion Makefile
Expand Up @@ -74,7 +74,7 @@ offlinetest: codetest
$(PYTHON) -m pytest -k "not download"

# XXX: This is hard to maintain
CODE_FOLDERS = yt_dlp yt_dlp/downloader yt_dlp/extractor yt_dlp/postprocessor yt_dlp/compat yt_dlp/utils yt_dlp/dependencies
CODE_FOLDERS = yt_dlp yt_dlp/downloader yt_dlp/extractor yt_dlp/postprocessor yt_dlp/compat yt_dlp/compat/urllib yt_dlp/utils yt_dlp/dependencies
yt-dlp: yt_dlp/*.py yt_dlp/*/*.py
mkdir -p zip
for d in $(CODE_FOLDERS) ; do \
Expand Down
2 changes: 1 addition & 1 deletion yt_dlp/YoutubeDL.py
Expand Up @@ -21,9 +21,9 @@
import tokenize
import traceback
import unicodedata
import urllib.request

from .cache import Cache
from .compat import urllib # isort: split
from .compat import compat_os_name, compat_shlex_quote
from .cookies import load_cookies
from .downloader import FFmpegFD, get_suitable_downloader, shorten_protocol_name
Expand Down
7 changes: 7 additions & 0 deletions yt_dlp/compat/urllib/__init__.py
@@ -0,0 +1,7 @@
# flake8: noqa: F405
from urllib import * # noqa: F403

pukkandan marked this conversation as resolved.
Show resolved Hide resolved
from ..compat_utils import passthrough_module

passthrough_module(__name__, 'urllib')
del passthrough_module
40 changes: 40 additions & 0 deletions yt_dlp/compat/urllib/request.py
@@ -0,0 +1,40 @@
# flake8: noqa: F405
from urllib.request import * # noqa: F403

from ..compat_utils import passthrough_module

passthrough_module(__name__, 'urllib.request')
del passthrough_module


from .. import compat_os_name

if compat_os_name == 'nt':
# On older python versions, proxies are extracted from Windows registry erroneously. [1]
# If the https proxy in the registry does not have a scheme, urllib will incorrectly add https:// to it. [2]
# It is unlikely that the user has actually set it to be https, so we should be fine to safely downgrade
# it to http on these older python versions to avoid issues
# This also applies for ftp proxy type, as ftp:// proxy scheme is not supported.
# 1: https://github.com/python/cpython/issues/86793
# 2: https://github.com/python/cpython/blob/51f1ae5ceb0673316c4e4b0175384e892e33cc6e/Lib/urllib/request.py#L2683-L2698
import sys
from urllib.request import getproxies_environment, getproxies_registry

def getproxies_registry_patched():
proxies = getproxies_registry()
if (
sys.version_info >= (3, 10, 5) # https://docs.python.org/3.10/whatsnew/changelog.html#python-3-10-5-final
or (3, 9, 13) <= sys.version_info < (3, 10) # https://docs.python.org/3.9/whatsnew/changelog.html#python-3-9-13-final
coletdjnz marked this conversation as resolved.
Show resolved Hide resolved
):
return proxies

for scheme in ('https', 'ftp'):
if scheme in proxies and proxies[scheme].startswith(f'{scheme}://'):
proxies[scheme] = 'http' + proxies[scheme][len(scheme):]

return proxies

def getproxies():
return getproxies_environment() or getproxies_registry_patched()

del compat_os_name