Skip to content

Commit

Permalink
[extractor/common] add convenient parameters to _download_*
Browse files Browse the repository at this point in the history
  • Loading branch information
@Lesmiscore
Lesmiscore committed Jan 7, 2022
1 parent 7f6c06d commit 303e3a6
Show file tree
Hide file tree
Showing 2 changed files with 66 additions and 30 deletions.
12 changes: 6 additions & 6 deletions yt_dlp/extractor/_youtube_download_services.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# automating YouTube downloading services to download deleted videos
import re

from ..compat import compat_urllib_parse_urlencode, compat_str
from ..compat import compat_str
from ..utils import (
ExtractorError,
int_or_none,
Expand Down Expand Up @@ -54,14 +54,14 @@ def _real_extract_normal(self, url):
video_id = self.BASE_IE._match_id(self.remove_prefix(url))
self._download_webpage('https://www.y2mate.com/youtube/%s' % video_id, video_id)
common_headers = {'X-Requested-With': 'XMLHttpRequest', 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8'}
request_data = urlencode_postdata({
request_data = {
'url': 'https://www.youtube.com/watch?v=%s' % video_id,
'q_auto': '1',
'ajax': '1'
})
}
size_specs = self._download_json(
'https://www.y2mate.com/mates/analyze/ajax', video_id,
note='Fetching size specs', errnote='This video is unavailable', data=request_data,
note='Fetching size specs', errnote='This video is unavailable', form_params=request_data,
headers=common_headers)
if size_specs.get('status') != 'success':
raise ExtractorError('Server responded with status %s' % size_specs.get('status'))
Expand Down Expand Up @@ -139,7 +139,7 @@ def _real_extract_normal(self, url):
for i in range(retries):
url_data = self._download_json(
'https://www.y2mate.com/mates/convert', video_id,
note='Fetching infomation for %s (%d of %d)' % (format_name, i + 1, retries), data=compat_urllib_parse_urlencode(request_data).encode('utf-8'),
note='Fetching infomation for %s (%d of %d)' % (format_name, i + 1, retries), form_params=request_data,
headers=common_headers)
if url_data.get('status') != 'success':
self.report_warning('Server responded with status %s' % url_data.get('status'))
Expand Down Expand Up @@ -217,7 +217,7 @@ def _real_extract(self, url):
}
response = self._download_json(
'https://www.clipconverter.cc/check.php', video_id, note='Requesting for extraction',
data=urlencode_postdata(post_data),
form_params=post_data,
headers={
'Origin': 'https://www.clipconverter.cc',
'Referer': 'https://www.clipconverter.cc/2/',
Expand Down
84 changes: 60 additions & 24 deletions yt_dlp/extractor/common.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@
update_url_query,
url_basename,
url_or_none,
urlencode_postdata,
urljoin,
variadic,
xpath_element,
Expand Down Expand Up @@ -707,12 +708,17 @@ def __can_accept_status_code(err, expected_status):
else:
return err.code in variadic(expected_status)

def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, data=None, headers={}, query={}, expected_status=None):
def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, data=None, headers=None, query=None, expected_status=None):
"""
Return the response handle.
See _download_webpage docstring for arguments specification.
"""

if not headers:
headers = {}
if not query:
query = {}
if not self._downloader._first_webpage_request:
sleep_interval = self.get_param('sleep_interval_requests') or 0
if sleep_interval > 0:
Expand Down Expand Up @@ -772,7 +778,11 @@ def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fa
self.report_warning(errmsg)
return False

def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True, encoding=None, data=None, headers={}, query={}, expected_status=None):
def _download_webpage_handle(
self, url_or_request, video_id, note=None, errnote=None, fatal=True,
encoding=None, data=None, headers={}, query={}, expected_status=None,
json_body=None, form_params=None, body_encoding=None):

"""
Return a tuple (page content as string, URL handle).
Expand All @@ -782,6 +792,16 @@ def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=
if isinstance(url_or_request, (compat_str, str)):
url_or_request = url_or_request.partition('#')[0]

# False is valid for JSON value ("false") while form_params isn't
if json_body is not None or form_params:
# both "data" and, "json_body" or "form_params" are specified
if data:
raise ExtractorError('Both "data" parameter and either "json_body" or "form_params" are specified')
if json_body is not None:
data = json.dumps(json_body).encode(body_encoding or encoding or 'utf-8')
elif form_params:
data = urlencode_postdata(form_params)

urlh = self._request_webpage(url_or_request, video_id, note, errnote, fatal, data=data, headers=headers, query=query, expected_status=expected_status)
if urlh is False:
assert not fatal
Expand Down Expand Up @@ -875,7 +895,8 @@ def _webpage_read_content(self, urlh, url_or_request, video_id, note=None, errno
def _download_webpage(
self, url_or_request, video_id, note=None, errnote=None,
fatal=True, tries=1, timeout=5, encoding=None, data=None,
headers={}, query={}, expected_status=None):
headers=None, query=None, expected_status=None,
json_body=None, form_params=None, body_encoding=None):
"""
Return the data of the page as a string.
Expand Down Expand Up @@ -917,7 +938,8 @@ def _download_webpage(
res = self._download_webpage_handle(
url_or_request, video_id, note, errnote, fatal,
encoding=encoding, data=data, headers=headers, query=query,
expected_status=expected_status)
expected_status=expected_status,
json_body=json_body, form_params=form_params, body_encoding=body_encoding)
success = True
except compat_http_client.IncompleteRead as e:
try_count += 1
Expand All @@ -933,8 +955,9 @@ def _download_webpage(
def _download_xml_handle(
self, url_or_request, video_id, note='Downloading XML',
errnote='Unable to download XML', transform_source=None,
fatal=True, encoding=None, data=None, headers={}, query={},
expected_status=None):
fatal=True, encoding=None, data=None, headers=None, query=None,
expected_status=None,
json_body=None, form_params=None, body_encoding=None):
"""
Return a tuple (xml as an compat_etree_Element, URL handle).
Expand All @@ -943,7 +966,8 @@ def _download_xml_handle(
res = self._download_webpage_handle(
url_or_request, video_id, note, errnote, fatal=fatal,
encoding=encoding, data=data, headers=headers, query=query,
expected_status=expected_status)
expected_status=expected_status, json_body=json_body,
form_params=form_params, body_encoding=body_encoding)
if res is False:
return res
xml_string, urlh = res
Expand All @@ -955,7 +979,8 @@ def _download_xml(
self, url_or_request, video_id,
note='Downloading XML', errnote='Unable to download XML',
transform_source=None, fatal=True, encoding=None,
data=None, headers={}, query={}, expected_status=None):
data=None, headers=None, query=None, expected_status=None,
json_body=None, form_params=None, body_encoding=None):
"""
Return the xml as an compat_etree_Element.
Expand All @@ -965,7 +990,8 @@ def _download_xml(
url_or_request, video_id, note=note, errnote=errnote,
transform_source=transform_source, fatal=fatal, encoding=encoding,
data=data, headers=headers, query=query,
expected_status=expected_status)
expected_status=expected_status,
json_body=json_body, form_params=form_params, body_encoding=body_encoding)
return res if res is False else res[0]

def _parse_xml(self, xml_string, video_id, transform_source=None, fatal=True):
Expand All @@ -984,8 +1010,9 @@ def _parse_xml(self, xml_string, video_id, transform_source=None, fatal=True):
def _download_json_handle(
self, url_or_request, video_id, note='Downloading JSON metadata',
errnote='Unable to download JSON metadata', transform_source=None,
fatal=True, encoding=None, data=None, headers={}, query={},
expected_status=None):
fatal=True, encoding=None, data=None, headers=None, query=None,
expected_status=None,
json_body=None, form_params=None, body_encoding=None):
"""
Return a tuple (JSON object, URL handle).
Expand All @@ -994,7 +1021,8 @@ def _download_json_handle(
res = self._download_webpage_handle(
url_or_request, video_id, note, errnote, fatal=fatal,
encoding=encoding, data=data, headers=headers, query=query,
expected_status=expected_status)
expected_status=expected_status,
json_body=json_body, form_params=form_params, body_encoding=body_encoding)
if res is False:
return res
json_string, urlh = res
Expand All @@ -1005,8 +1033,9 @@ def _download_json_handle(
def _download_json(
self, url_or_request, video_id, note='Downloading JSON metadata',
errnote='Unable to download JSON metadata', transform_source=None,
fatal=True, encoding=None, data=None, headers={}, query={},
expected_status=None):
fatal=True, encoding=None, data=None, headers=None, query=None,
expected_status=None,
json_body=None, form_params=None, body_encoding=None):
"""
Return the JSON object as a dict.
Expand All @@ -1016,7 +1045,8 @@ def _download_json(
url_or_request, video_id, note=note, errnote=errnote,
transform_source=transform_source, fatal=fatal, encoding=encoding,
data=data, headers=headers, query=query,
expected_status=expected_status)
expected_status=expected_status,
json_body=json_body, form_params=form_params, body_encoding=body_encoding)
return res if res is False else res[0]

def _parse_json(self, json_string, video_id, transform_source=None, fatal=True):
Expand All @@ -1039,8 +1069,9 @@ def _parse_socket_response_as_json(self, data, video_id, transform_source=None,
def _download_socket_json_handle(
self, url_or_request, video_id, note='Polling socket',
errnote='Unable to poll socket', transform_source=None,
fatal=True, encoding=None, data=None, headers={}, query={},
expected_status=None):
fatal=True, encoding=None, data=None, headers=None, query=None,
expected_status=None,
json_body=None, form_params=None, body_encoding=None):
"""
Return a tuple (JSON object, URL handle).
Expand All @@ -1049,7 +1080,8 @@ def _download_socket_json_handle(
res = self._download_webpage_handle(
url_or_request, video_id, note, errnote, fatal=fatal,
encoding=encoding, data=data, headers=headers, query=query,
expected_status=expected_status)
expected_status=expected_status,
json_body=json_body, form_params=form_params, body_encoding=body_encoding)
if res is False:
return res
webpage, urlh = res
Expand All @@ -1060,8 +1092,9 @@ def _download_socket_json_handle(
def _download_socket_json(
self, url_or_request, video_id, note='Polling socket',
errnote='Unable to poll socket', transform_source=None,
fatal=True, encoding=None, data=None, headers={}, query={},
expected_status=None):
fatal=True, encoding=None, data=None, headers=None, query=None,
expected_status=None,
json_body=None, form_params=None, body_encoding=None):
"""
Return the JSON object as a dict.
Expand All @@ -1071,7 +1104,8 @@ def _download_socket_json(
url_or_request, video_id, note=note, errnote=errnote,
transform_source=transform_source, fatal=fatal, encoding=encoding,
data=data, headers=headers, query=query,
expected_status=expected_status)
expected_status=expected_status,
json_body=json_body, form_params=form_params, body_encoding=body_encoding)
return res if res is False else res[0]

def report_warning(self, msg, video_id=None, *args, only_once=False, **kwargs):
Expand Down Expand Up @@ -2124,14 +2158,16 @@ def _extract_m3u8_formats(self, *args, **kwargs):
def _extract_m3u8_formats_and_subtitles(
self, m3u8_url, video_id, ext=None, entry_protocol='m3u8_native',
preference=None, quality=None, m3u8_id=None, note=None,
errnote=None, fatal=True, live=False, data=None, headers={},
query={}):
errnote=None, fatal=True, live=False, data=None, headers=None,
query=None,
json_body=None, form_params=None, body_encoding=None):

res = self._download_webpage_handle(
m3u8_url, video_id,
note='Downloading m3u8 information' if note is None else note,
errnote='Failed to download m3u8 information' if errnote is None else errnote,
fatal=fatal, data=data, headers=headers, query=query)
fatal=fatal, data=data, headers=headers, query=query,
json_body=json_body, form_params=form_params, body_encoding=body_encoding)

if res is False:
return [], {}
Expand Down

0 comments on commit 303e3a6

Please sign in to comment.