[core] Rework HTTP networking

Makefile

@@ -76,7 +76,7 @@ offlinetest: codetest
 
 # XXX: This is hard to maintain
 CODE_FOLDERS = yt_dlp yt_dlp/downloader yt_dlp/extractor yt_dlp/postprocessor yt_dlp/compat \
-               yt_dlp/extractor/anvato_token_generator
+               yt_dlp/compat/urllib yt_dlp/extractor/anvato_token_generator yt_dlp/networking
 yt-dlp: yt_dlp/*.py yt_dlp/*/*.py
 	mkdir -p zip
 	for d in $(CODE_FOLDERS) ; do \

README.md

@@ -453,6 +453,8 @@ You can also fork the project on github and run your fork's [build workflow](.gi
     --source-address IP             Client-side IP address to bind to
     -4, --force-ipv4                Make all connections via IPv4
     -6, --force-ipv6                Make all connections via IPv6
+    --enable-file-protocol          Enable file:// URLs. This is disabled by
+                                    default for security reasons.
 
 ## Geo-restriction:
     --geo-verification-proxy URL    Use this proxy to verify the IP address for

test/test_YoutubeDL.py

@@ -10,7 +10,6 @@
 
 import copy
 import json
-import urllib.error
 
 from test.helper import FakeYDL, assertRegexpMatches
 from yt_dlp import YoutubeDL
@@ -25,6 +24,7 @@
     int_or_none,
     match_filter_func,
 )
+from yt_dlp.networking.exceptions import RequestError
 
 TEST_URL = 'http://localhost/sample.mp4'
 
@@ -1100,7 +1100,7 @@ def test_selection(params, expected_ids, evaluate_all=False):
     def test_urlopen_no_file_protocol(self):
         # see https://github.com/ytdl-org/youtube-dl/issues/8227
         ydl = YDL()
-        self.assertRaises(urllib.error.URLError, ydl.urlopen, 'file:///etc/passwd')
+        self.assertRaises(RequestError, ydl.urlopen, 'file:///etc/passwd')
 
     def test_do_not_override_ie_key_in_url_transparent(self):
         ydl = YDL()

test/test_YoutubeDLCookieJar.py

@@ -11,7 +11,7 @@
 import re
 import tempfile
 
-from yt_dlp.utils import YoutubeDLCookieJar
+from yt_dlp.cookies import YoutubeDLCookieJar
 
 
 class TestYoutubeDLCookieJar(unittest.TestCase):

test/test_download.py

@@ -10,10 +10,7 @@
 
 import collections
 import hashlib
-import http.client
 import json
-import socket
-import urllib.error
 
 from test.helper import (
     assertGreaterEqual,
@@ -26,7 +23,6 @@
     report_warning,
     try_rm,
 )
-
 import yt_dlp.YoutubeDL  # isort: split
 from yt_dlp.extractor import get_info_extractor
 from yt_dlp.utils import (
@@ -36,6 +32,7 @@
     format_bytes,
     join_nonempty,
 )
+from yt_dlp.networking.exceptions import TransportError, HTTPError
 
 RETRIES = 3
 
@@ -161,8 +158,7 @@ def try_rm_tcs_files(tcs=None):
                         force_generic_extractor=params.get('force_generic_extractor', False))
                 except (DownloadError, ExtractorError) as err:
                     # Check if the exception is not a network related one
-                    if (err.exc_info[0] not in (urllib.error.URLError, socket.timeout, UnavailableVideoError, http.client.BadStatusLine)
-                            or (err.exc_info[0] == urllib.error.HTTPError and err.exc_info[1].code == 503)):
+                    if not isinstance(err.exc_info[1], (TransportError, UnavailableVideoError)) or (isinstance(err.exc_info[1], HTTPError) and err.exc_info[1].code == 503):
                         err.msg = f'{getattr(err, "msg", err)} ({tname})'
                         raise
 
@@ -246,7 +242,7 @@ def try_rm_tcs_files(tcs=None):
                 # extractor returns full results even with extract_flat
                 res_tcs = [{'info_dict': e} for e in res_dict['entries']]
                 try_rm_tcs_files(res_tcs)
-
+            ydl.close()
     return test_template