From fe6249d7ea08a28e58b42ddd093c7cc1d19213b7 Mon Sep 17 00:00:00 2001 From: coletdjnz Date: Sun, 21 May 2023 13:00:05 +1200 Subject: [PATCH 1/3] [core] Workaround erroneous urllib Windows proxy parsing Authored by: coletdjnz --- yt_dlp/YoutubeDL.py | 3 ++- yt_dlp/compat/urllib/__init__.py | 7 +++++++ yt_dlp/compat/urllib/request.py | 36 ++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 yt_dlp/compat/urllib/__init__.py create mode 100644 yt_dlp/compat/urllib/request.py diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index 1162d2df1ae..3a0db85a36a 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -25,6 +25,7 @@ from .cache import Cache from .compat import compat_os_name, compat_shlex_quote +from .compat.urllib.request import getproxies from .cookies import load_cookies from .downloader import FFmpegFD, get_suitable_downloader, shorten_protocol_name from .downloader.rtmp import rtmpdump_version @@ -3890,7 +3891,7 @@ def _setup_opener(self): else: proxies = {'http': opts_proxy, 'https': opts_proxy} else: - proxies = urllib.request.getproxies() + proxies = getproxies() # Set HTTPS proxy to HTTP one if given (https://github.com/ytdl-org/youtube-dl/issues/805) if 'http' in proxies and 'https' not in proxies: proxies['https'] = proxies['http'] diff --git a/yt_dlp/compat/urllib/__init__.py b/yt_dlp/compat/urllib/__init__.py new file mode 100644 index 00000000000..6b6b8e103df --- /dev/null +++ b/yt_dlp/compat/urllib/__init__.py @@ -0,0 +1,7 @@ +# flake8: noqa: F405 +from urllib import * # noqa: F403 + +from ..compat_utils import passthrough_module + +passthrough_module(__name__, 'urllib') +del passthrough_module diff --git a/yt_dlp/compat/urllib/request.py b/yt_dlp/compat/urllib/request.py new file mode 100644 index 00000000000..5d2da90e48c --- /dev/null +++ b/yt_dlp/compat/urllib/request.py @@ -0,0 +1,36 @@ +# flake8: noqa: F405 +from urllib.request import * # noqa: F403 +from .. import compat_os_name +from ..compat_utils import passthrough_module +import sys + +passthrough_module(__name__, 'urllib.request') +del passthrough_module + + +if compat_os_name == 'nt': + # On older python versions, proxies are extracted from Windows registry erroneously. [1] + # If the https proxy in the registry does not have a scheme, urllib will incorrectly add https:// to it. [2] + # It is unlikely that the user has actually set it to be https, so we should be fine to safely downgrade + # it to http on these older python versions to avoid issues + # This also applies for ftp proxy type, as ftp:// proxy scheme is not supported. + # 1: https://github.com/python/cpython/issues/86793 + # 2: https://github.com/python/cpython/blob/51f1ae5ceb0673316c4e4b0175384e892e33cc6e/Lib/urllib/request.py#L2683-L2698 + from urllib.request import getproxies_environment, getproxies_registry + + def getproxies_registry_patched(): + proxies = getproxies_registry() + if ( + sys.version_info >= (3, 10, 5) # https://docs.python.org/3.10/whatsnew/changelog.html#python-3-10-5-final + or (3, 9, 13) <= sys.version_info < (3, 10) # https://docs.python.org/3.9/whatsnew/changelog.html#python-3-9-13-final + ): + return proxies + + for scheme in ('https', 'ftp'): + if scheme in proxies and proxies[scheme].startswith(f'{scheme}://'): + proxies[scheme] = 'http' + proxies[scheme][len(scheme):] + + return proxies + + def getproxies(): + return getproxies_environment() or getproxies_registry_patched() From f14184341a719260845b14e16ba7f1d6cb5b6b64 Mon Sep 17 00:00:00 2001 From: coletdjnz Date: Sun, 21 May 2023 15:59:14 +1200 Subject: [PATCH 2/3] Add code folder to make file --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f03fe205239..b1ac0e7d684 100644 --- a/Makefile +++ b/Makefile @@ -74,7 +74,7 @@ offlinetest: codetest $(PYTHON) -m pytest -k "not download" # XXX: This is hard to maintain -CODE_FOLDERS = yt_dlp yt_dlp/downloader yt_dlp/extractor yt_dlp/postprocessor yt_dlp/compat yt_dlp/utils yt_dlp/dependencies +CODE_FOLDERS = yt_dlp yt_dlp/downloader yt_dlp/extractor yt_dlp/postprocessor yt_dlp/compat yt_dlp/compat/urllib yt_dlp/utils yt_dlp/dependencies yt-dlp: yt_dlp/*.py yt_dlp/*/*.py mkdir -p zip for d in $(CODE_FOLDERS) ; do \ From 2754f7bdc2136fca7cac12da0ba6b84a5b0a2ebe Mon Sep 17 00:00:00 2001 From: pukkandan Date: Mon, 22 May 2023 18:10:47 +0530 Subject: [PATCH 3/3] Simplify import --- yt_dlp/YoutubeDL.py | 5 ++--- yt_dlp/compat/urllib/request.py | 8 ++++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index 3a0db85a36a..585b0411d53 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -21,11 +21,10 @@ import tokenize import traceback import unicodedata -import urllib.request from .cache import Cache +from .compat import urllib # isort: split from .compat import compat_os_name, compat_shlex_quote -from .compat.urllib.request import getproxies from .cookies import load_cookies from .downloader import FFmpegFD, get_suitable_downloader, shorten_protocol_name from .downloader.rtmp import rtmpdump_version @@ -3891,7 +3890,7 @@ def _setup_opener(self): else: proxies = {'http': opts_proxy, 'https': opts_proxy} else: - proxies = getproxies() + proxies = urllib.request.getproxies() # Set HTTPS proxy to HTTP one if given (https://github.com/ytdl-org/youtube-dl/issues/805) if 'http' in proxies and 'https' not in proxies: proxies['https'] = proxies['http'] diff --git a/yt_dlp/compat/urllib/request.py b/yt_dlp/compat/urllib/request.py index 5d2da90e48c..ff63b2f0e9f 100644 --- a/yt_dlp/compat/urllib/request.py +++ b/yt_dlp/compat/urllib/request.py @@ -1,13 +1,14 @@ # flake8: noqa: F405 from urllib.request import * # noqa: F403 -from .. import compat_os_name + from ..compat_utils import passthrough_module -import sys passthrough_module(__name__, 'urllib.request') del passthrough_module +from .. import compat_os_name + if compat_os_name == 'nt': # On older python versions, proxies are extracted from Windows registry erroneously. [1] # If the https proxy in the registry does not have a scheme, urllib will incorrectly add https:// to it. [2] @@ -16,6 +17,7 @@ # This also applies for ftp proxy type, as ftp:// proxy scheme is not supported. # 1: https://github.com/python/cpython/issues/86793 # 2: https://github.com/python/cpython/blob/51f1ae5ceb0673316c4e4b0175384e892e33cc6e/Lib/urllib/request.py#L2683-L2698 + import sys from urllib.request import getproxies_environment, getproxies_registry def getproxies_registry_patched(): @@ -34,3 +36,5 @@ def getproxies_registry_patched(): def getproxies(): return getproxies_environment() or getproxies_registry_patched() + +del compat_os_name