From 8fcb513cf1193cef74d7943faf00c4eb8c50a1f1 Mon Sep 17 00:00:00 2001 From: Boo1098 Date: Mon, 22 Apr 2024 21:24:46 -0700 Subject: [PATCH] Add Page Size Overrides per Channel This is a minimum viable product. Tested all 3 overrides and they worked. The current method of resetting the override is clunk (setting to negative number). I've also upended some of the build query in subscriptions and haven't fully tested if that messes with things. --- .../home/src/download/subscriptions.py | 65 ++++++++++++++----- tubearchivist/home/src/frontend/forms.py | 9 +++ tubearchivist/home/src/index/channel.py | 15 +++-- .../home/templates/home/channel_id_about.html | 45 +++++++++++-- 4 files changed, 111 insertions(+), 23 deletions(-) diff --git a/tubearchivist/home/src/download/subscriptions.py b/tubearchivist/home/src/download/subscriptions.py index c6d85a9c..e73c7cda 100644 --- a/tubearchivist/home/src/download/subscriptions.py +++ b/tubearchivist/home/src/download/subscriptions.py @@ -38,7 +38,11 @@ def get_channels(subscribed_only=True): return all_channels def get_last_youtube_videos( - self, channel_id, limit=True, query_filter=VideoTypeEnum.UNKNOWN + self, + channel_id, + limit=True, + query_filter=VideoTypeEnum.UNKNOWN, + channel_overwrites=None, ): """get a list of last videos from channel""" queries = self._build_queries(query_filter, limit) @@ -50,18 +54,50 @@ def get_last_youtube_videos( "skip_download": True, "extract_flat": True, } + + vid_type = vid_type_enum.value + + limit_amount = 0 + + if channel_overwrites: + if ( + vid_type == "videos" + and "subscriptions_channel_size" in channel_overwrites + ): + limit_amount = channel_overwrites[ + "subscriptions_channel_size" + ] + if ( + vid_type == "shorts" + and "subscriptions_shorts_channel_size" + in channel_overwrites + ): + limit_amount = channel_overwrites[ + "subscriptions_shorts_channel_size" + ] + if ( + vid_type == "streams" + and "subscriptions_live_channel_size" in channel_overwrites + ): + limit_amount = channel_overwrites[ + "subscriptions_live_channel_size" + ] + if limit: obs["playlistend"] = limit_amount - vid_type = vid_type_enum.value - channel = YtWrap(obs, self.config).extract( - f"https://www.youtube.com/channel/{channel_id}/{vid_type}" - ) - if not channel: - continue - last_videos.extend( - [(i["id"], i["title"], vid_type) for i in channel["entries"]] - ) + if not limit or limit_amount > 0: + channel_query = YtWrap(obs, self.config).extract( + f"https://www.youtube.com/channel/{channel_id}/{vid_type}" + ) + if not channel_query: + continue + last_videos.extend( + [ + (i["id"], i["title"], vid_type) + for i in channel_query["entries"] + ] + ) return last_videos @@ -86,10 +122,6 @@ def _build_queries(self, query_filter, limit): return queries for query_item, default_limit in limit_map.items(): - if not default_limit: - # is deactivated in config - continue - if limit: query_limit = default_limit else: @@ -115,7 +147,10 @@ def find_missing(self): for idx, channel in enumerate(all_channels): channel_id = channel["channel_id"] print(f"{channel_id}: find missing videos.") - last_videos = self.get_last_youtube_videos(channel_id) + last_videos = self.get_last_youtube_videos( + channel_id, + channel_overwrites=channel.get("channel_overwrites"), + ) if last_videos: for video_id, _, vid_type in last_videos: diff --git a/tubearchivist/home/src/frontend/forms.py b/tubearchivist/home/src/frontend/forms.py index f8fcb798..118e0dc8 100644 --- a/tubearchivist/home/src/frontend/forms.py +++ b/tubearchivist/home/src/frontend/forms.py @@ -303,3 +303,12 @@ class ChannelOverwriteForm(forms.Form): integrate_sponsorblock = forms.ChoiceField( widget=forms.Select, choices=SP_CHOICES, required=False ) + subscriptions_channel_size = forms.IntegerField( + label=False, required=False + ) + subscriptions_live_channel_size = forms.IntegerField( + label=False, required=False + ) + subscriptions_shorts_channel_size = forms.IntegerField( + label=False, required=False + ) diff --git a/tubearchivist/home/src/index/channel.py b/tubearchivist/home/src/index/channel.py index 08166f70..99dfcbbe 100644 --- a/tubearchivist/home/src/index/channel.py +++ b/tubearchivist/home/src/index/channel.py @@ -345,23 +345,30 @@ def set_overwrites(self, overwrites): "autodelete_days", "index_playlists", "integrate_sponsorblock", + "subscriptions_channel_size", + "subscriptions_live_channel_size", + "subscriptions_shorts_channel_size", ] to_write = self.json_data.get("channel_overwrites", {}) for key, value in overwrites.items(): if key not in valid_keys: raise ValueError(f"invalid overwrite key: {key}") - if value == "disable": + elif value == "disable": to_write[key] = False continue - if value in [0, "0"]: + elif value == "0": if key in to_write: del to_write[key] continue - if value == "1": + elif value == "1": to_write[key] = True continue - if value: + elif isinstance(value, int) and int(value) < 0: + if key in to_write: + del to_write[key] + continue + elif value is not None and value != "": to_write.update({key: value}) self.json_data["channel_overwrites"] = to_write diff --git a/tubearchivist/home/templates/home/channel_id_about.html b/tubearchivist/home/templates/home/channel_id_about.html index cd965f31..56fae3dc 100644 --- a/tubearchivist/home/templates/home/channel_id_about.html +++ b/tubearchivist/home/templates/home/channel_id_about.html @@ -107,16 +107,20 @@

Customize {{ channel_info.channel_name }}

{{ channel_info.channel_overwrites.download_format }} {% else %} False - {% endif %}

+ {% endif %}
+ Enter "disable" to disable this override. +

{{ channel_overwrite_form.download_format }}

Auto delete watched videos after x days: - {% if channel_info.channel_overwrites.autodelete_days %} + {% if channel_info.channel_overwrites.autodelete_days is not None %} {{ channel_info.channel_overwrites.autodelete_days }} {% else %} False - {% endif %}

+ {% endif %}
+ Enter a negative number to disable this override. +

{{ channel_overwrite_form.autodelete_days }}
@@ -139,6 +143,39 @@

Customize {{ channel_info.channel_name }}

{% endif %}

{{ channel_overwrite_form.integrate_sponsorblock }}
+

Page Size Overrides


+

Disable standard videos, shorts, or streams for this channel by setting their page size to 0 (zero).


+

Disable page size overwrite for channel by setting to negative value.


+
+

YouTube page size: + {% if channel_info.channel_overwrites.subscriptions_channel_size is not None %} + {{ channel_info.channel_overwrites.subscriptions_channel_size }} + {% else %} + False + {% endif %}

+ Videos to scan to find new items for the Rescan subscriptions task, max recommended 50.
+ {{ channel_overwrite_form.subscriptions_channel_size }}
+
+
+

YouTube Live page size: + {% if channel_info.channel_overwrites.subscriptions_live_channel_size is not None %} + {{ channel_info.channel_overwrites.subscriptions_live_channel_size }} + {% else %} + False + {% endif %}

+ Live Videos to scan to find new items for the Rescan subscriptions task, max recommended 50.
+ {{ channel_overwrite_form.subscriptions_live_channel_size }}
+
+
+

YouTube Shorts page size: + {% if channel_info.channel_overwrites.subscriptions_shorts_channel_size is not None %} + {{ channel_info.channel_overwrites.subscriptions_shorts_channel_size }} + {% else %} + False + {% endif %}

+ Shorts Videos to scan to find new items for the Rescan subscriptions task, max recommended 50.
+ {{ channel_overwrite_form.subscriptions_shorts_channel_size }}
+

@@ -146,4 +183,4 @@

Customize {{ channel_info.channel_name }}

{% endif %} -{% endblock content %} \ No newline at end of file +{% endblock content %}