improve yt proxying

.gitignore

@@ -145,6 +145,7 @@ dmypy.json
 .pytype/
 
 # Added
+yotter-config.yaml
 data_export.json
 .env
 .envo

app/routes.py

@@ -1,5 +1,6 @@
 import datetime
 import glob
+import yaml
 import json
 import math
 import os
@@ -39,7 +40,7 @@
 ##########################
 #### Config variables ####
 ##########################
-config = json.load(open('yotter-config.json'))
+config = yaml.safe_load(open('yotter-config.yaml'))
 ##########################
 #### Config variables ####
 ##########################
@@ -335,15 +336,11 @@ def ytsearch():
             prev_page = "/ytsearch?q={q}&s={s}&p={p}".format(q=query, s=sort, p=int(page) - 1)
 
         for video in results['videos']:
-            hostname = urllib.parse.urlparse(video['videoThumb']).netloc
-            video['videoThumb'] = video['videoThumb'].replace("https://{}".format(hostname), "") + "&host=" + hostname
+            video['videoThumb'] = proxy_image_url(video['videoThumb'])
 
         for channel in results['channels']:
-            if config['nginxVideoStream']:
-                channel['thumbnail'] = channel['thumbnail'].replace("~", "/")
-                hostName = urllib.parse.urlparse(channel['thumbnail']).netloc
-                channel['thumbnail'] = channel['thumbnail'].replace("https://{}".format(hostName),
-                                                                    "") + "?host=" + hostName
+            channel['thumbnail'] = proxy_image_url(channel['thumbnail'])
+
         return render_template('ytsearch.html', form=form, btform=button_form, results=results,
                                restricted=config['restrictPublicUsage'], config=config, npage=next_page,
                                ppage=prev_page)
@@ -423,18 +420,9 @@ def channel(id):
     data = ytch.get_channel_tab_info(id, page, sort)
 
     for video in data['items']:
-        if config['nginxVideoStream']:
-            hostName = urllib.parse.urlparse(video['thumbnail'][1:]).netloc
-            video['thumbnail'] = video['thumbnail'].replace("https://{}".format(hostName), "")[1:].replace("hqdefault",
-                                                                                                       "mqdefault") + "&host=" + hostName
-        else:
-            video['thumbnail'] = video['thumbnail'].replace('/', '~')
+        video['thumbnail'] = proxy_image_url(video['thumbnail'])
 
-    if config['nginxVideoStream']:
-        hostName = urllib.parse.urlparse(data['avatar'][1:]).netloc
-        data['avatar'] = data['avatar'].replace("https://{}".format(hostName), "")[1:] + "?host=" + hostName
-    else:
-        data['avatar'] = data['avatar'].replace('/', '~')
+    data['avatar'] = proxy_image_url(data['avatar'])
 
     next_page = "/channel/{q}?s={s}&p={p}".format(q=id, s=sort, p=int(page) + 1)
     if int(page) == 1:
@@ -482,13 +470,11 @@ def watch():
         retry -= 1
 
     for source in vsources:
-        hostName = urllib.parse.urlparse(source['src']).netloc
-        source['src'] = source['src'].replace("https://{}".format(hostName), "") + "&host=" + hostName
+        source['src'] = proxy_video_source_url(source['src'])
 
     # Parse video formats
     for v_format in info['formats']:
-        hostName = urllib.parse.urlparse(v_format['url']).netloc
-        v_format['url'] = v_format['url'].replace("https://{}".format(hostName), "") + "&host=" + hostName
+        v_format['url'] = proxy_video_source_url(v_format['url'])
         if v_format['audio_bitrate'] is not None and v_format['vcodec'] is None:
             v_format['audio_valid'] = True
 
@@ -503,6 +489,8 @@ def watch():
     videocomments = utils.post_process_comments_info(videocomments)
     if videocomments is not None:
         videocomments.sort(key=lambda x: x['likes'], reverse=True)
+        for cmnt in videocomments:
+            cmnt['thumbnail'] = proxy_image_url(cmnt['thumbnail'])
 
     # Calculate rating %
     if info['like_count']+info['dislike_count']>0:
@@ -523,11 +511,10 @@ def markupString(string):
 
 
 ## PROXY videos through Yotter server to the client.
-@app.route('/stream/<url>', methods=['GET', 'POST'])
+@app.route('/stream/<path:url>', methods=['GET', 'POST'])
 @login_required
 def stream(url):
     # This function proxies the video stream from GoogleVideo to the client.
-    url = url.replace('YotterSlash', '/')
     headers = Headers()
     if (url):
         s = requests.Session()
@@ -553,6 +540,16 @@ def download_file(streamable):
         for chunk in stream.iter_content(chunk_size=8192):
             yield chunk
 
+# Proxy yt images through server
+@app.route('/ytimg/<path:url>')
+@login_required
+def ytimg(url):
+    pic = requests.get(url, stream=True)
+    response = Response(pic, mimetype=pic.headers['Content-Type'], direct_passthrough=True)
+    # extend browser file caching with etags (ytimg uses 7200)
+    response.cache_control.public = True
+    response.cache_control.max_age = int(60000)
+    return response
 
 #########################
 #### General Logic ######
@@ -585,6 +582,22 @@ def img(url):
     pic = requests.get(url.replace("~", "/"))
     return Response(pic, mimetype="image/png")
 
+def proxy_url(url, endpoint, config_entry):
+    if not config.get(config_entry, None): return url
+    ext_proxy = config.get('external_proxy', None)
+    if ext_proxy:
+        parsed = urllib.parse.urlparse(url)._asdict()
+        parsed['url'] = url
+        encoded = {key+'_encoded': urllib.parse.quote_plus(value) for (key,value) in parsed.items()}
+        joined = dict(parsed, **encoded)
+        return ext_proxy.format(**joined)
+    return url_for(endpoint,url=url)
+
+def proxy_video_source_url(url):
+    return proxy_url(url, 'stream', 'proxy_videos')
+
+def proxy_image_url(url):
+    return proxy_url(url, 'ytimg', 'proxy_images').replace('hqdefault','mqdefault')
 
 @app.route('/logout')
 def logout():
@@ -1027,12 +1040,8 @@ def getYoutubePosts(ids):
                 video.channelUrl = vid.author_detail.href
                 video.id = vid.yt_videoid
                 video.videoTitle = vid.title
-                if config['nginxVideoStream']:
-                    hostName = urllib.parse.urlparse(vid.media_thumbnail[0]['url']).netloc
-                    video.videoThumb = vid.media_thumbnail[0]['url'].replace("https://{}".format(hostName), "").replace(
-                        "hqdefault", "mqdefault") + "?host=" + hostName
-                else:
-                    video.videoThumb = vid.media_thumbnail[0]['url'].replace('/', '~')
+                video.videoThumb = proxy_image_url(vid.media_thumbnail[0]['url'])
+
                 video.views = vid.media_statistics['views']
                 video.description = vid.summary_detail.value
                 video.description = re.sub(r'^https?:\/\/.*[\r\n]*', '', video.description[0:120] + "...",

app/templates/video.html

@@ -40,11 +40,9 @@ <h5 class="ui header">Livestreams are under developent and still not supported o
             controls
             buffered
             preload="none">
-            {% if config.nginxVideoStream %}
-                {% for source in vsources %}
-                        <source src="{{source.src}}" type="{{source.type}}">
-                {% endfor %}
-            {% endif %}
+            {% for source in vsources %}
+                    <source src="{{source.src}}" type="{{source.type}}">
+            {% endfor %}
             </video-js>
         </div>
     {%endif%}
@@ -115,7 +113,7 @@ <h3 class="ui dividing header">Comments</h3>
                 {% include '_video_comment.html' %}
         {% endfor %}
     </div>
-            
+
 <script src="{{ url_for('static',filename='video.min.js') }}"></script>
 {% if info.live %}
     <script src="{{ url_for('static',filename='videojs-http-streaming.min.js')}}"></script>

app/templates/ytsearch.html

@@ -19,15 +19,11 @@
     {% if results.channels %}
         <h3 class="ui dividing header">Users</h3>
     {% endif %}
-    <div class="ui relaxed divided list">        
+    <div class="ui relaxed divided list">
             {% for res in results.channels %}
                     <div class="item">
                         <div class="image">
-                            {% if config.nginxVideoStream %}
-                                <img src="{{res.thumbnail}}" alt="Avatar">
-                            {% else %}    
-                                <img alt="Avatar" src="{{ url_for('img', url=res.thumbnail) }}">
-                            {% endif %}
+                            <img src="{{res.thumbnail}}" alt="Avatar">
                         </div>
                         <div class="content">
                           <a class = "header" href="{{ url_for('channel', id=res.channelId)}}">{{res.username}}</a>
@@ -45,7 +41,7 @@ <h3 class="ui dividing header">Users</h3>
                             <div class="ui label">
                                 <i class="video icon"></i> {{res.videos}}
                             </div>
-                            
+
                             {% if restricted or current_user.is_authenticated %}
                                 <div class="right floated content">
                                 {% if not current_user.is_following_yt(res.channelId) %}
@@ -59,7 +55,7 @@ <h3 class="ui dividing header">Users</h3>
                                         {{ btform.submit(value='Unfollow') }}
                                     </form>
                                 {% endif %}
-                                </div>  
+                                </div>
                             {% endif %}
                           </div>
                         </div>

requirements.txt

@@ -41,6 +41,7 @@ pylint==2.6.0
 PyMySQL==0.10.1
 pyparsing==2.4.7
 PySocks==1.7.1
+pyyaml==5.3.1
 python-dateutil==2.8.1
 python-dotenv==0.14.0
 python-editor==1.0.4

yotter-config.sample.yaml

@@ -0,0 +1,37 @@
+
+serverName: "yotter.xyz"
+nitterInstance: "https://nitter.net/"
+maxInstanceUsers: 100
+serverLocation: "Germany"
+restrictPublicUsage: true
+
+# Deprecated
+#nginxVideoStream: true
+
+# Whether to proxy images (thumbnails etc.) through the server
+proxy_images: true
+
+# Whether to proxy video (or audio) streams through the server
+proxy_videos: true
+
+# Url for external proxy server
+# If this is unset and `proxy_*` is set to true, requests will be proxied internally.
+# It's recommended to use an external proxy such as nginx for better performance.
+#
+# It's possible to use variables in `{}` that refer to the original request url:
+# `scheme`, `netloc` (host:port), `path` (with a leading `/`), `query` (without leading `?`), `fragment` (without leading `#`): as per `urllib.parse.urlparse`
+# `scheme_encoded`, `netloc_encoded` etc.: same as above but urlencoded
+# `url`, `url_encoded`: the original request url, and urlencoded variant
+#
+# The example below works for the nginx configuration in `SELF-HOSTING.md`
+#
+#external_proxy: "https://my.nginx.instance{path}?{query}&host={netloc}"
+
+maintenance_mode: false
+show_admin_message: false
+admin_message_title: "Message from the admin"
+admin_message: "Message from the admin text"
+admin_user: "admin_username"
+max_old_user_days: 60
+donate_url: ""
+

youtube/channel.py

@@ -180,29 +180,19 @@ def get_channel_search_json(channel_id, query, page):
     ctoken = base64.urlsafe_b64encode(proto.nested(80226972, ctoken)).decode('ascii')
 
     polymer_json = util.fetch_url("https://www.youtube.com/browse_ajax?ctoken=" + ctoken, headers_desktop, debug_name='channel_search')
-
     return polymer_json
 
-
 def post_process_channel_info(info):
-    info['avatar'] = util.prefix_url(info['avatar'])
-    info['channel_url'] = util.prefix_url(info['channel_url'])
     for item in info['items']:
-        util.prefix_urls(item)
         util.add_extra_html_info(item)
 
-
-
-
-
 playlist_sort_codes = {'2': "da", '3': "dd", '4': "lad"}
 
 # youtube.com/[channel_id]/[tab]
 # youtube.com/user/[username]/[tab]
 # youtube.com/c/[custom]/[tab]
 # youtube.com/[custom]/[tab]
 def get_channel_page_general_url(base_url, tab, request, channel_id=None):
-
     page_number = int(request.args.get('page', 1))
     sort = request.args.get('sort', '3')
     view = request.args.get('view', '1')

youtube/channels.py

@@ -172,7 +172,7 @@ def get_author_info_from_channel(content):
     channel = {
         "channelId": cmd['channelId'],
         "username": cmd['title'],
-        "thumbnail": "https:{}".format(cmd['avatar']['thumbnails'][0]['url'].replace("/", "~")),
+        "thumbnail": "https:{}".format(cmd['avatar']['thumbnails'][0]['url']),
         "description":description,
         "suscribers": cmd['subscriberCountText']['runs'][0]['text'].split(" ")[0],
         "banner": cmd['banner']['thumbnails'][0]['url']

youtube/playlist.py

@@ -14,15 +14,15 @@
 
 
 
-def playlist_ctoken(playlist_id, offset):  
-    
+def playlist_ctoken(playlist_id, offset):
+
     offset = proto.uint(1, offset)
     # this is just obfuscation as far as I can tell. It doesn't even follow protobuf
     offset = b'PT:' + proto.unpadded_b64encode(offset)
     offset = proto.string(15, offset)
 
     continuation_info = proto.string( 3, proto.percent_b64encode(offset) )
-    
+
     playlist_id = proto.string(2, 'VL' + playlist_id )
     pointless_nest = proto.string(80226972, playlist_id + continuation_info)
 
@@ -51,7 +51,7 @@ def playlist_first_page(playlist_id, report_text = "Retrieved playlist"):
     content = json.loads(util.uppercase_escape(content.decode('utf-8')))
 
     return content
-    
+
 
 #https://m.youtube.com/playlist?itct=CBMQybcCIhMIptj9xJaJ2wIV2JKcCh3Idwu-&ctoken=4qmFsgI2EiRWTFBMT3kwajlBdmxWWlB0bzZJa2pLZnB1MFNjeC0tN1BHVEMaDmVnWlFWRHBEUWxFJTNE&pbj=1
 def get_videos(playlist_id, page):
@@ -84,7 +84,7 @@ def get_playlist_page():
         this_page_json = first_page_json
     else:
         tasks = (
-            gevent.spawn(playlist_first_page, playlist_id, report_text="Retrieved playlist info" ), 
+            gevent.spawn(playlist_first_page, playlist_id, report_text="Retrieved playlist info" ),
             gevent.spawn(get_videos, playlist_id, page)
         )
         gevent.joinall(tasks)
@@ -98,12 +98,10 @@ def get_playlist_page():
     if page != '1':
         info['metadata'] = yt_data_extract.extract_playlist_metadata(first_page_json)
 
-    util.prefix_urls(info['metadata'])
     for item in info.get('items', ()):
-        util.prefix_urls(item)
         util.add_extra_html_info(item)
         if 'id' in item:
-            item['thumbnail'] = '/https://i.ytimg.com/vi/' + item['id'] + '/default.jpg'
+            item['thumbnail'] = 'https://i.ytimg.com/vi/' + item['id'] + '/default.jpg'
 
         item['url'] += '&list=' + playlist_id
         if item['index']: