Skip to content

Commit

Permalink
[CI] auto update yt_dlp to upstream commit 4ccd73fea0f6f4be343e1ec7f2…
Browse files Browse the repository at this point in the history 
…2dd03799addcf8
  • Loading branch information
@github-actions
github-actions[bot] committed May 21, 2024
1 parent d2a8697 commit e65fb37
Show file tree
Hide file tree
Showing 3 changed files with 123 additions and 42 deletions.
153 changes: 115 additions & 38 deletions lib/yt_dlp/extractor/tiktok.py
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import functools
import itertools
import json
import random
import re
import string
import time
import uuid

Expand All @@ -15,10 +15,13 @@
UnsupportedError,
UserNotLive,
determine_ext,
filter_dict,
format_field,
int_or_none,
join_nonempty,
merge_dicts,
mimetype2ext,
parse_qs,
qualities,
remove_start,
srt_subtitles_timecode,
Expand Down Expand Up @@ -49,11 +52,21 @@ class TikTokBaseIE(InfoExtractor):
_APP_INFO = None
_APP_USER_AGENT = None

@property
@functools.cached_property
def _KNOWN_APP_INFO(self):
return self._configuration_arg('app_info', ie_key=TikTokIE)
# If we have a genuine device ID, we may not need any IID
default = [''] if self._KNOWN_DEVICE_ID else []
return self._configuration_arg('app_info', default, ie_key=TikTokIE)

@property
@functools.cached_property
def _KNOWN_DEVICE_ID(self):
return self._configuration_arg('device_id', [None], ie_key=TikTokIE)[0]

@functools.cached_property
def _DEVICE_ID(self):
return self._KNOWN_DEVICE_ID or str(random.randint(7250000000000000000, 7351147085025500000))

@functools.cached_property
def _API_HOSTNAME(self):
return self._configuration_arg(
'api_hostname', ['api16-normal-c-useast1a.tiktokv.com'], ie_key=TikTokIE)[0]
Expand Down Expand Up @@ -115,7 +128,7 @@ def _call_api_impl(self, ep, query, video_id, fatal=True,
}, query=query)

def _build_api_query(self, query):
return {
return filter_dict({
**query,
'device_platform': 'android',
'os': 'android',
Expand Down Expand Up @@ -156,10 +169,10 @@ def _build_api_query(self, query):
'build_number': self._APP_INFO['app_version'],
'region': 'US',
'ts': int(time.time()),
'iid': self._APP_INFO['iid'],
'device_id': random.randint(7250000000000000000, 7351147085025500000),
'iid': self._APP_INFO.get('iid'),
'device_id': self._DEVICE_ID,
'openudid': ''.join(random.choices('0123456789abcdef', k=16)),
}
})

def _call_api(self, ep, query, video_id, fatal=True,
note='Downloading API JSON', errnote='Unable to download API page'):
Expand Down Expand Up @@ -239,23 +252,22 @@ def _get_subtitles(self, aweme_detail, aweme_id):
})
return subtitles

def _parse_url_key(self, url_key):
format_id, codec, res, bitrate = self._search_regex(
r'v[^_]+_(?P<id>(?P<codec>[^_]+)_(?P<res>\d+p)_(?P<bitrate>\d+))', url_key,
'url key', default=(None, None, None, None), group=('id', 'codec', 'res', 'bitrate'))
if not format_id:
return {}, None
return {
'format_id': format_id,
'vcodec': 'h265' if codec == 'bytevc1' else codec,
'tbr': int_or_none(bitrate, scale=1000) or None,
'quality': qualities(self.QUALITIES)(res),
}, res

def _parse_aweme_video_app(self, aweme_detail):
aweme_id = aweme_detail['aweme_id']
video_info = aweme_detail['video']

def parse_url_key(url_key):
format_id, codec, res, bitrate = self._search_regex(
r'v[^_]+_(?P<id>(?P<codec>[^_]+)_(?P<res>\d+p)_(?P<bitrate>\d+))', url_key,
'url key', default=(None, None, None, None), group=('id', 'codec', 'res', 'bitrate'))
if not format_id:
return {}, None
return {
'format_id': format_id,
'vcodec': 'h265' if codec == 'bytevc1' else codec,
'tbr': int_or_none(bitrate, scale=1000) or None,
'quality': qualities(self.QUALITIES)(res),
}, res

known_resolutions = {}

def audio_meta(url):
Expand All @@ -270,7 +282,7 @@ def audio_meta(url):
} if ext == 'mp3' or '-music-' in url else {}

def extract_addr(addr, add_meta={}):
parsed_meta, res = parse_url_key(addr.get('url_key', ''))
parsed_meta, res = self._parse_url_key(addr.get('url_key', ''))
is_bytevc2 = parsed_meta.get('vcodec') == 'bytevc2'
if res:
known_resolutions.setdefault(res, {}).setdefault('height', int_or_none(addr.get('height')))
Expand All @@ -284,7 +296,7 @@ def extract_addr(addr, add_meta={}):
'acodec': 'aac',
'source_preference': -2 if 'aweme/v1' in url else -1, # Downloads from API might get blocked
**add_meta, **parsed_meta,
# bytevc2 is bytedance's proprietary (unplayable) video codec
# bytevc2 is bytedance's own custom h266/vvc codec, as-of-yet unplayable
'preference': -100 if is_bytevc2 else -1,
'format_note': join_nonempty(
add_meta.get('format_note'), '(API)' if 'aweme/v1' in url else None,
Expand All @@ -296,6 +308,7 @@ def extract_addr(addr, add_meta={}):
formats = []
width = int_or_none(video_info.get('width'))
height = int_or_none(video_info.get('height'))
ratio = try_call(lambda: width / height) or 0.5625
if video_info.get('play_addr'):
formats.extend(extract_addr(video_info['play_addr'], {
'format_id': 'play_addr',
Expand All @@ -312,8 +325,8 @@ def extract_addr(addr, add_meta={}):
'format_id': 'download_addr',
'format_note': 'Download video%s' % (', watermarked' if video_info.get('has_watermark') else ''),
'vcodec': 'h264',
'width': dl_width or width,
'height': try_call(lambda: int(dl_width / 0.5625)) or height, # download_addr['height'] is wrong
'width': dl_width,
'height': try_call(lambda: int(dl_width / ratio)), # download_addr['height'] is wrong
'preference': -2 if video_info.get('has_watermark') else -1,
}))
if video_info.get('play_addr_h264'):
Expand Down Expand Up @@ -420,26 +433,88 @@ def _parse_aweme_video_web(self, aweme_detail, webpage_url, video_id):
formats = []
width = int_or_none(video_info.get('width'))
height = int_or_none(video_info.get('height'))
ratio = try_call(lambda: width / height) or 0.5625
COMMON_FORMAT_INFO = {
'ext': 'mp4',
'vcodec': 'h264',
'acodec': 'aac',
}

for bitrate_info in traverse_obj(video_info, ('bitrateInfo', lambda _, v: v['PlayAddr']['UrlList'])):
format_info, res = self._parse_url_key(
traverse_obj(bitrate_info, ('PlayAddr', 'UrlKey', {str})) or '')
# bytevc2 is bytedance's own custom h266/vvc codec, as-of-yet unplayable
is_bytevc2 = format_info.get('vcodec') == 'bytevc2'
format_info.update({
'format_note': 'UNPLAYABLE' if is_bytevc2 else None,
'preference': -100 if is_bytevc2 else -1,
'filesize': traverse_obj(bitrate_info, ('PlayAddr', 'DataSize', {int_or_none})),
})

if dimension := (res and int(res[:-1])):
if dimension == 540: # '540p' is actually 576p
dimension = 576
if ratio < 1: # portrait: res/dimension is width
y = int(dimension / ratio)
format_info.update({
'width': dimension,
'height': y - (y % 2),
})
else: # landscape: res/dimension is height
x = int(dimension * ratio)
format_info.update({
'width': x - (x % 2),
'height': dimension,
})

for video_url in traverse_obj(bitrate_info, ('PlayAddr', 'UrlList', ..., {url_or_none})):
formats.append({
**COMMON_FORMAT_INFO,
**format_info,
'url': self._proto_relative_url(video_url),
})

# We don't have res string for play formats, but need quality for sorting & de-duplication
play_quality = traverse_obj(formats, (lambda _, v: v['width'] == width, 'quality', any))

for play_url in traverse_obj(video_info, ('playAddr', ((..., 'src'), None), {url_or_none})):
formats.append({
**COMMON_FORMAT_INFO,
'format_id': 'play',
'url': self._proto_relative_url(play_url),
'ext': 'mp4',
'width': width,
'height': height,
'quality': play_quality,
})

for download_url in traverse_obj(video_info, (('downloadAddr', ('download', 'url')), {url_or_none})):
formats.append({
**COMMON_FORMAT_INFO,
'format_id': 'download',
'url': self._proto_relative_url(download_url),
'ext': 'mp4',
'width': width,
'height': height,
})

self._remove_duplicate_formats(formats)

for f in traverse_obj(formats, lambda _, v: 'unwatermarked' not in v['url']):
f.update({
'format_note': join_nonempty(f.get('format_note'), 'watermarked', delim=', '),
'preference': f.get('preference') or -2,
})

# Is it a slideshow with only audio for download?
if not formats and traverse_obj(music_info, ('playUrl', {url_or_none})):
audio_url = music_info['playUrl']
ext = traverse_obj(parse_qs(audio_url), (
'mime_type', -1, {lambda x: x.replace('_', '/')}, {mimetype2ext})) or 'm4a'
formats.append({
'format_id': 'audio',
'url': self._proto_relative_url(audio_url),
'ext': ext,
'acodec': 'aac' if ext == 'm4a' else ext,
'vcodec': 'none',
})

thumbnails = []
for thumb_url in traverse_obj(aweme_detail, (
(None, 'video'), ('thumbnail', 'cover', 'dynamicCover', 'originCover'), {url_or_none})):
Expand All @@ -451,10 +526,17 @@ def _parse_aweme_video_web(self, aweme_detail, webpage_url, video_id):

return {
'id': video_id,
**traverse_obj(music_info, {
'track': ('title', {str}),
'album': ('album', {str}, {lambda x: x or None}),
'artists': ('authorName', {str}, {lambda x: [x] if x else None}),
'duration': ('duration', {int_or_none}),
}),
**traverse_obj(aweme_detail, {
'title': ('desc', {str}),
'description': ('desc', {str}),
'duration': ('video', 'duration', {int_or_none}),
# audio-only slideshows have a video duration of 0 and an actual audio duration
'duration': ('video', 'duration', {int_or_none}, {lambda x: x or None}),
'timestamp': ('createTime', {int_or_none}),
}),
**traverse_obj(author_info or aweme_detail, {
Expand All @@ -469,11 +551,6 @@ def _parse_aweme_video_web(self, aweme_detail, webpage_url, video_id):
'repost_count': 'shareCount',
'comment_count': 'commentCount',
}, expected_type=int_or_none),
**traverse_obj(music_info, {
'track': ('title', {str}),
'album': ('album', {str}, {lambda x: x or None}),
'artists': ('authorName', {str}, {lambda x: [x] if x else None}),
}),
'channel_id': channel_id,
'uploader_url': user_url,
'formats': formats,
Expand Down Expand Up @@ -848,7 +925,7 @@ def _video_entries_api(self, webpage, user_id, username):
'max_cursor': 0,
'min_cursor': 0,
'retry_type': 'no_retry',
'device_id': ''.join(random.choices(string.digits, k=19)), # Some endpoints don't like randomized device_id, so it isn't directly set in _call_api.
'device_id': self._DEVICE_ID, # Some endpoints don't like randomized device_id, so it isn't directly set in _call_api.
}

for page in itertools.count(1):
Expand Down Expand Up @@ -896,7 +973,7 @@ def _entries(self, list_id, display_id):
'cursor': 0,
'count': 20,
'type': 5,
'device_id': ''.join(random.choices(string.digits, k=19))
'device_id': self._DEVICE_ID,
}

for page in itertools.count(1):
Expand Down
10 changes: 7 additions & 3 deletions lib/yt_dlp/networking/_requests.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@
if urllib3_version < (1, 26, 17):
raise ImportError('Only urllib3 >= 1.26.17 is supported')

if requests.__build__ < 0x023100:
raise ImportError('Only requests >= 2.31.0 is supported')
if requests.__build__ < 0x023200:
raise ImportError('Only requests >= 2.32.0 is supported')

import requests.adapters
import requests.utils
Expand Down Expand Up @@ -181,9 +181,13 @@ def proxy_manager_for(self, proxy, **proxy_kwargs):
return super().proxy_manager_for(proxy, **proxy_kwargs, **self._pm_args, **extra_kwargs)

def cert_verify(*args, **kwargs):
# lean on SSLContext for cert verification
# Lean on our SSLContext for cert verification
pass

def _get_connection(self, request, *_, proxies=None, **__):
# Lean on our SSLContext for cert verification
return self.get_connection(request.url, proxies)


class RequestsSession(requests.sessions.Session):
"""
Expand Down
2 changes: 1 addition & 1 deletion lib/yt_dlp_version
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
3e35aa32c74bc108375be8c8b6b3bfc90dfff1b4
4ccd73fea0f6f4be343e1ec7f22dd03799addcf8

0 comments on commit e65fb37

Please sign in to comment.