Skip to content

Commit

Permalink
Add Page Size Overrides per Channel
Browse files Browse the repository at this point in the history 
This is a minimum viable product. Tested all 3 overrides and they
worked. The current method of resetting the override is clunk (setting
to negative number). I've also upended some of the build query in
subscriptions and haven't fully tested if that messes with things.
  • Loading branch information
@Boo1098
Boo1098 committed Apr 27, 2024
1 parent 33ecd73 commit 8fcb513
Show file tree
Hide file tree
Showing 4 changed files with 111 additions and 23 deletions.
65 changes: 50 additions & 15 deletions tubearchivist/home/src/download/subscriptions.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,11 @@ def get_channels(subscribed_only=True):
return all_channels

def get_last_youtube_videos(
self, channel_id, limit=True, query_filter=VideoTypeEnum.UNKNOWN
self,
channel_id,
limit=True,
query_filter=VideoTypeEnum.UNKNOWN,
channel_overwrites=None,
):
"""get a list of last videos from channel"""
queries = self._build_queries(query_filter, limit)
Expand All @@ -50,18 +54,50 @@ def get_last_youtube_videos(
"skip_download": True,
"extract_flat": True,
}

vid_type = vid_type_enum.value

limit_amount = 0

if channel_overwrites:
if (
vid_type == "videos"
and "subscriptions_channel_size" in channel_overwrites
):
limit_amount = channel_overwrites[
"subscriptions_channel_size"
]
if (
vid_type == "shorts"
and "subscriptions_shorts_channel_size"
in channel_overwrites
):
limit_amount = channel_overwrites[
"subscriptions_shorts_channel_size"
]
if (
vid_type == "streams"
and "subscriptions_live_channel_size" in channel_overwrites
):
limit_amount = channel_overwrites[
"subscriptions_live_channel_size"
]

if limit:
obs["playlistend"] = limit_amount

vid_type = vid_type_enum.value
channel = YtWrap(obs, self.config).extract(
f"https://www.youtube.com/channel/{channel_id}/{vid_type}"
)
if not channel:
continue
last_videos.extend(
[(i["id"], i["title"], vid_type) for i in channel["entries"]]
)
if not limit or limit_amount > 0:
channel_query = YtWrap(obs, self.config).extract(
f"https://www.youtube.com/channel/{channel_id}/{vid_type}"
)
if not channel_query:
continue
last_videos.extend(
[
(i["id"], i["title"], vid_type)
for i in channel_query["entries"]
]
)

return last_videos

Expand All @@ -86,10 +122,6 @@ def _build_queries(self, query_filter, limit):
return queries

for query_item, default_limit in limit_map.items():
if not default_limit:
# is deactivated in config
continue

if limit:
query_limit = default_limit
else:
Expand All @@ -115,7 +147,10 @@ def find_missing(self):
for idx, channel in enumerate(all_channels):
channel_id = channel["channel_id"]
print(f"{channel_id}: find missing videos.")
last_videos = self.get_last_youtube_videos(channel_id)
last_videos = self.get_last_youtube_videos(
channel_id,
channel_overwrites=channel.get("channel_overwrites"),
)

if last_videos:
for video_id, _, vid_type in last_videos:
Expand Down
9 changes: 9 additions & 0 deletions tubearchivist/home/src/frontend/forms.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -303,3 +303,12 @@ class ChannelOverwriteForm(forms.Form):
integrate_sponsorblock = forms.ChoiceField(
widget=forms.Select, choices=SP_CHOICES, required=False
)
subscriptions_channel_size = forms.IntegerField(
label=False, required=False
)
subscriptions_live_channel_size = forms.IntegerField(
label=False, required=False
)
subscriptions_shorts_channel_size = forms.IntegerField(
label=False, required=False
)
15 changes: 11 additions & 4 deletions tubearchivist/home/src/index/channel.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -345,23 +345,30 @@ def set_overwrites(self, overwrites):
"autodelete_days",
"index_playlists",
"integrate_sponsorblock",
"subscriptions_channel_size",
"subscriptions_live_channel_size",
"subscriptions_shorts_channel_size",
]

to_write = self.json_data.get("channel_overwrites", {})
for key, value in overwrites.items():
if key not in valid_keys:
raise ValueError(f"invalid overwrite key: {key}")
if value == "disable":
elif value == "disable":
to_write[key] = False
continue
if value in [0, "0"]:
elif value == "0":
if key in to_write:
del to_write[key]
continue
if value == "1":
elif value == "1":
to_write[key] = True
continue
if value:
elif isinstance(value, int) and int(value) < 0:
if key in to_write:
del to_write[key]
continue
elif value is not None and value != "":
to_write.update({key: value})

self.json_data["channel_overwrites"] = to_write
Expand Down
45 changes: 41 additions & 4 deletions tubearchivist/home/templates/home/channel_id_about.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,16 +107,20 @@ <h2>Customize {{ channel_info.channel_name }}</h2>
{{ channel_info.channel_overwrites.download_format }}
{% else %}
False
{% endif %}</span></p>
{% endif %}</span><br>
Enter "disable" to disable this override.
</p>
{{ channel_overwrite_form.download_format }}<br>
</div>
<div class="overwrite-form-item">
<p>Auto delete watched videos after x days: <span class="settings-current">
{% if channel_info.channel_overwrites.autodelete_days %}
{% if channel_info.channel_overwrites.autodelete_days is not None %}
{{ channel_info.channel_overwrites.autodelete_days }}
{% else %}
False
{% endif %}</span></p>
{% endif %}</span><br>
Enter a negative number to disable this override.
</p>
{{ channel_overwrite_form.autodelete_days }}<br>
</div>
<div class="overwrite-form-item">
Expand All @@ -139,11 +143,44 @@ <h2>Customize {{ channel_info.channel_name }}</h2>
{% endif %}</span></p>
{{ channel_overwrite_form.integrate_sponsorblock }}<br>
</div>
<h3>Page Size Overrides</h3><br>
<p>Disable standard videos, shorts, or streams for this channel by setting their page size to 0 (zero).</p><br>
<p>Disable page size overwrite for channel by setting to negative value.</p><br>
<div class="overwrite-form-item">
<p>YouTube page size: <span class="settings-current">
{% if channel_info.channel_overwrites.subscriptions_channel_size is not None %}
{{ channel_info.channel_overwrites.subscriptions_channel_size }}
{% else %}
False
{% endif %}</span></p>
<i>Videos to scan to find new items for the <b>Rescan subscriptions</b> task, max recommended 50.</i><br>
{{ channel_overwrite_form.subscriptions_channel_size }}<br>
</div>
<div class="overwrite-form-item">
<p>YouTube Live page size: <span class="settings-current">
{% if channel_info.channel_overwrites.subscriptions_live_channel_size is not None %}
{{ channel_info.channel_overwrites.subscriptions_live_channel_size }}
{% else %}
False
{% endif %}</span></p>
<i>Live Videos to scan to find new items for the <b>Rescan subscriptions</b> task, max recommended 50.</i><br>
{{ channel_overwrite_form.subscriptions_live_channel_size }}<br>
</div>
<div class="overwrite-form-item">
<p>YouTube Shorts page size: <span class="settings-current">
{% if channel_info.channel_overwrites.subscriptions_shorts_channel_size is not None %}
{{ channel_info.channel_overwrites.subscriptions_shorts_channel_size }}
{% else %}
False
{% endif %}</span></p>
<i>Shorts Videos to scan to find new items for the <b>Rescan subscriptions</b> task, max recommended 50.</i><br>
{{ channel_overwrite_form.subscriptions_shorts_channel_size }}<br>
</div><br>
<button type="submit">Save Channel Overwrites</button>
</form>
</div>
</div>
{% endif %}
</div>
<script type="text/javascript" src="{% static 'progress.js' %}"></script>
{% endblock content %}
{% endblock content %}

0 comments on commit 8fcb513

Please sign in to comment.