Twitch support

tubesync/sync/admin.py

@@ -16,10 +16,14 @@ class SourceAdmin(admin.ModelAdmin):
 class MediaAdmin(admin.ModelAdmin):
 
     ordering = ('-created',)
-    list_display = ('uuid', 'key', 'source', 'can_download', 'skip', 'downloaded')
+    list_display = ('uuid', 'key', 'source', 'can_download', 'skip', 'is_live', 'downloaded')
     readonly_fields = ('uuid', 'created')
     search_fields = ('uuid', 'source__key', 'key')
 
+    @admin.display(boolean=True)
+    def is_live(self, obj):
+        return obj.is_live
+
 
 @admin.register(MediaServer)
 class MediaServerAdmin(admin.ModelAdmin):

tubesync/sync/migrations/0018_twitch_support.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.18 on 2023-03-28 15:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sync', '0017_alter_source_sponsorblock_categories'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='media',
+            name='last_crawl',
+            field=models.DateTimeField(blank=True, db_index=True, help_text='Date and time the metadata of the media was last crawled', null=True, verbose_name='last crawl'),
+        ),
+        migrations.AlterField(
+            model_name='source',
+            name='source_type',
+            field=models.CharField(choices=[('c', 'YouTube channel'), ('i', 'YouTube channel by ID'), ('p', 'YouTube playlist'), ('t', 'Twitch channel')], db_index=True, default='c', help_text='Source type', max_length=1, verbose_name='source type'),
+        ),
+    ]

tubesync/sync/models.py

@@ -14,8 +14,9 @@
 from common.errors import NoFormatException
 from common.utils import clean_filename
 from .youtube import (get_media_info as get_youtube_media_info,
-                      download_media as download_youtube_media)
-from .utils import seconds_to_timestr, parse_media_format
+                      download_media as download_youtube_media,
+                      get_twitch_media_info)
+from .utils import seconds_to_timestr, parse_media_format, parse_twitch_media_format
 from .matching import (get_best_combined_format, get_best_audio_format, 
                        get_best_video_format)
 from .mediaservers import PlexMediaServer
@@ -32,12 +33,14 @@ class Source(models.Model):
     SOURCE_TYPE_YOUTUBE_CHANNEL = 'c'
     SOURCE_TYPE_YOUTUBE_CHANNEL_ID = 'i'
     SOURCE_TYPE_YOUTUBE_PLAYLIST = 'p'
+    SOURCE_TYPE_TWITCH_CHANNEL = 't'
     SOURCE_TYPES = (SOURCE_TYPE_YOUTUBE_CHANNEL, SOURCE_TYPE_YOUTUBE_CHANNEL_ID,
-                    SOURCE_TYPE_YOUTUBE_PLAYLIST)
+                    SOURCE_TYPE_YOUTUBE_PLAYLIST, SOURCE_TYPE_TWITCH_CHANNEL)
     SOURCE_TYPE_CHOICES = (
         (SOURCE_TYPE_YOUTUBE_CHANNEL, _('YouTube channel')),
         (SOURCE_TYPE_YOUTUBE_CHANNEL_ID, _('YouTube channel by ID')),
         (SOURCE_TYPE_YOUTUBE_PLAYLIST, _('YouTube playlist')),
+        (SOURCE_TYPE_TWITCH_CHANNEL, _('Twitch channel')),
     )
 
     SOURCE_RESOLUTION_360P = '360p'
@@ -151,30 +154,35 @@ class Source(models.Model):
         SOURCE_TYPE_YOUTUBE_CHANNEL: '<i class="fab fa-youtube"></i>',
         SOURCE_TYPE_YOUTUBE_CHANNEL_ID: '<i class="fab fa-youtube"></i>',
         SOURCE_TYPE_YOUTUBE_PLAYLIST: '<i class="fab fa-youtube"></i>',
+        SOURCE_TYPE_TWITCH_CHANNEL: '<i class="fab fa-twitch"></i>',
     }
     # Format to use to display a URL for the source
     URLS = {
         SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/c/{key}',
         SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/channel/{key}',
         SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/playlist?list={key}',
+        SOURCE_TYPE_TWITCH_CHANNEL: 'https://www.twitch.tv/{key}/videos',
     }
     # Format used to create indexable URLs
     INDEX_URLS = {
         SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/c/{key}/videos',
         SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/channel/{key}/videos',
         SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/playlist?list={key}',
+        SOURCE_TYPE_TWITCH_CHANNEL: 'https://www.twitch.tv/{key}/videos?filter=all&sort=time',
     }
     # Callback functions to get a list of media from the source
     INDEXERS = {
         SOURCE_TYPE_YOUTUBE_CHANNEL: get_youtube_media_info,
         SOURCE_TYPE_YOUTUBE_CHANNEL_ID: get_youtube_media_info,
         SOURCE_TYPE_YOUTUBE_PLAYLIST: get_youtube_media_info,
+        SOURCE_TYPE_TWITCH_CHANNEL: get_twitch_media_info,
     }
     # Field names to find the media ID used as the key when storing media
     KEY_FIELD = {
         SOURCE_TYPE_YOUTUBE_CHANNEL: 'id',
         SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'id',
         SOURCE_TYPE_YOUTUBE_PLAYLIST: 'id',
+        SOURCE_TYPE_TWITCH_CHANNEL: 'id',
     }
 
     class CapChoices(models.IntegerChoices):
@@ -546,29 +554,41 @@ class Media(models.Model):
         Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/watch?v={key}',
         Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/watch?v={key}',
         Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/watch?v={key}',
+        Source.SOURCE_TYPE_TWITCH_CHANNEL: 'https://www.twitch.tv/videos/{key}',
     }
     # Callback functions to get a list of media from the source
     INDEXERS = {
         Source.SOURCE_TYPE_YOUTUBE_CHANNEL: get_youtube_media_info,
         Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: get_youtube_media_info,
         Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: get_youtube_media_info,
+        Source.SOURCE_TYPE_TWITCH_CHANNEL: get_youtube_media_info,
+    }
+    # Callback functions to get a list of media from the source
+    FORMAT_PARSER = {
+        Source.SOURCE_TYPE_YOUTUBE_CHANNEL: parse_media_format,
+        Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: parse_media_format,
+        Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: parse_media_format,
+        Source.SOURCE_TYPE_TWITCH_CHANNEL: parse_twitch_media_format,
     }
     # Maps standardised names to names used in source metdata
     METADATA_FIELDS = {
         'upload_date': {
             Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'upload_date',
             Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'upload_date',
             Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'upload_date',
+            Source.SOURCE_TYPE_TWITCH_CHANNEL: 'upload_date',
         },
         'title': {
             Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'title',
             Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'title',
             Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'title',
+            Source.SOURCE_TYPE_TWITCH_CHANNEL: 'title',
         },
         'thumbnail': {
             Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'thumbnail',
             Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'thumbnail',
             Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'thumbnail',
+            Source.SOURCE_TYPE_TWITCH_CHANNEL: 'thumbnail',
         },
         'description': {
             Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'description',
@@ -579,53 +599,69 @@ class Media(models.Model):
             Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'duration',
             Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'duration',
             Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'duration',
+            Source.SOURCE_TYPE_TWITCH_CHANNEL: 'duration',
         },
         'formats': {
             Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'formats',
             Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'formats',
             Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'formats',
+            Source.SOURCE_TYPE_TWITCH_CHANNEL: 'formats',
         },
         'categories': {
             Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'categories',
             Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'categories',
             Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'categories',
+            Source.SOURCE_TYPE_TWITCH_CHANNEL: 'categories',
         },
         'rating': {
             Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'average_rating',
             Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'average_rating',
             Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'average_rating',
+            Source.SOURCE_TYPE_TWITCH_CHANNEL: 'average_rating',
         },
         'age_limit': {
             Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'age_limit',
             Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'age_limit',
             Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'age_limit',
+            Source.SOURCE_TYPE_TWITCH_CHANNEL: 'age_limit',
         },
         'uploader': {
             Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'uploader',
             Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'uploader',
             Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'uploader',
+            Source.SOURCE_TYPE_TWITCH_CHANNEL: 'uploader',
         },
         'upvotes': {
             Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'like_count',
             Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'like_count',
             Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'like_count',
+            Source.SOURCE_TYPE_TWITCH_CHANNEL: 'like_count',
         },
         'downvotes': {
             Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'dislike_count',
             Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'dislike_count',
             Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'dislike_count',
+            Source.SOURCE_TYPE_TWITCH_CHANNEL: 'dislike_count',
         },
         'playlist_title': {
             Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'playlist_title',
             Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'playlist_title',
             Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'playlist_title',
+            Source.SOURCE_TYPE_TWITCH_CHANNEL: 'playlist_title',
+        },
+        'is_live': {
+            Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'is_live',
+            Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'is_live',
+            Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'is_live',
+            Source.SOURCE_TYPE_TWITCH_CHANNEL: 'is_live',
         },
     }
     STATE_UNKNOWN = 'unknown'
     STATE_SCHEDULED = 'scheduled'
     STATE_DOWNLOADING = 'downloading'
     STATE_DOWNLOADED = 'downloaded'
     STATE_SKIPPED = 'skipped'
+    STATE_LIVE = 'live'
     STATE_DISABLED_AT_SOURCE = 'source-disabled'
     STATE_ERROR = 'error'
     STATES = (STATE_UNKNOWN, STATE_SCHEDULED, STATE_DOWNLOADING, STATE_DOWNLOADED,
@@ -638,6 +674,7 @@ class Media(models.Model):
         STATE_SKIPPED: '<i class="fas fa-exclamation-circle" title="Skipped"></i>',
         STATE_DISABLED_AT_SOURCE: '<i class="fas fa-stop-circle" title="Media downloading disabled at source"></i>',
         STATE_ERROR: '<i class="fas fa-exclamation-triangle" title="Error downloading"></i>',
+        STATE_LIVE: '<i class="fas fa-clock" title="Live, download postponed"></i>',
     }
 
     uuid = models.UUIDField(
@@ -798,6 +835,13 @@ class Media(models.Model):
         null=True,
         help_text=_('Size of the downloaded media in bytes')
     )
+    last_crawl = models.DateTimeField(
+        _('last crawl'),
+        db_index=True,
+        null=True,
+        blank=True,
+        help_text=_('Date and time the metadata of the media was last crawled')
+    )
 
     def __str__(self):
         return self.key
@@ -814,8 +858,9 @@ def get_metadata_field(self, field):
         return fields.get(self.source.source_type, '')
 
     def iter_formats(self):
+        parser = self.FORMAT_PARSER.get(self.source.source_type)
         for fmt in self.formats:
-            yield parse_media_format(fmt)
+            yield parser(fmt)
 
     def get_best_combined_format(self):
         return get_best_combined_format(self)
@@ -1018,13 +1063,20 @@ def url(self):
 
     @property
     def description(self):
+        if self.source.source_type == Source.SOURCE_TYPE_TWITCH_CHANNEL:
+            chapters = self.loaded_metadata.get('chapters', [])
+            descriptions = [chapter['title'] for chapter in chapters]
+            return ' - '.join(descriptions)
+
         field = self.get_metadata_field('description')
-        return self.loaded_metadata.get(field, '').strip()
+        description = self.loaded_metadata.get(field) or ''
+        return description.strip()
 
     @property
     def title(self):
         field = self.get_metadata_field('title')
-        return self.loaded_metadata.get(field, '').strip()
+        title = self.loaded_metadata.get(field) or ''
+        return title.strip()
 
     @property
     def slugtitle(self):
@@ -1034,7 +1086,8 @@ def slugtitle(self):
     @property
     def thumbnail(self):
         field = self.get_metadata_field('thumbnail')
-        return self.loaded_metadata.get(field, '').strip()
+        thumbnail = self.loaded_metadata.get(field) or ''
+        return thumbnail.strip()
 
     @property
     def name(self):
@@ -1207,6 +1260,13 @@ def content_type(self):
         else:
             return 'video/mp4'
 
+    @property
+    def is_live(self):
+        field = self.get_metadata_field('is_live')
+        is_live = self.loaded_metadata.get(field)
+        is_live = True if is_live else False
+        return is_live
+
     @property
     def nfoxml(self):
         '''
@@ -1318,6 +1378,8 @@ def get_download_state(self, task=None):
             return self.STATE_SKIPPED
         if not self.source.download_media:
             return self.STATE_DISABLED_AT_SOURCE
+        if self.is_live:
+            return self.STATE_LIVE
         return self.STATE_UNKNOWN
 
     def get_download_state_icon(self, task=None):

tubesync/sync/signals.py

@@ -1,8 +1,10 @@
+from datetime import timedelta
 import os
 from django.conf import settings
 from django.db.models.signals import pre_save, post_save, pre_delete, post_delete
 from django.dispatch import receiver
 from django.utils.translation import gettext_lazy as _
+from django.utils import timezone
 from background_task.signals import task_failed
 from background_task.models import Task
 from common.logger import log
@@ -150,20 +152,28 @@ def media_post_save(sender, instance, created, **kwargs):
         post_save.disconnect(media_post_save, sender=Media)
         instance.save()
         post_save.connect(media_post_save, sender=Media)
-    # If the media is missing metadata schedule it to be downloaded
-    if not instance.metadata:
-        log.info(f'Scheduling task to download metadata for: {instance.url}')
+    # If the media is missing metadata schedule it to be downloaded,
+    # or refresh it if it was live during last run
+    if not instance.metadata or instance.is_live:
+        kwargs = {}
+        if instance.is_live:
+            kwargs['schedule'] = timedelta(hours=1)
+
+            log.info(f'Recheduling task to download metadata for live stream in 1h: {instance.url}')
+        else:
+            log.info(f'Scheduling task to download metadata for: {instance.url}')
         verbose_name = _('Downloading metadata for "{}"')
         download_media_metadata(
             str(instance.pk),
             priority=10,
             verbose_name=verbose_name.format(instance.pk),
-            remove_existing_tasks=True
+            remove_existing_tasks=True,
+            **kwargs
         )
     # If the media is missing a thumbnail schedule it to be downloaded
     if not instance.thumb_file_exists:
         instance.thumb = None
-    if not instance.thumb:
+    if not instance.thumb and not instance.is_live:
         thumbnail_url = instance.thumbnail
         if thumbnail_url:
             log.info(f'Scheduling task to download thumbnail for: {instance.name} '
@@ -177,12 +187,13 @@ def media_post_save(sender, instance, created, **kwargs):
                 verbose_name=verbose_name.format(instance.name),
                 remove_existing_tasks=True
             )
+
     # If the media has not yet been downloaded schedule it to be downloaded
     if not instance.media_file_exists:
         instance.downloaded = False
         instance.media_file = None
     if (not instance.downloaded and instance.can_download and not instance.skip
-        and instance.source.download_media):
+        and instance.source.download_media and not instance.is_live):
         delete_task_by_media('sync.tasks.download_media', (str(instance.pk),))
         verbose_name = _('Downloading media for "{}"')
         download_media(

tubesync/sync/tasks.py

@@ -229,6 +229,7 @@ def download_media_metadata(media_id):
     source = media.source
     metadata = media.index_metadata()
     media.metadata = json.dumps(metadata, default=json_serial)
+    media.last_crawl = timezone.now()
     upload_date = media.upload_date
     # Media must have a valid upload date
     if upload_date:

tubesync/sync/templates/sync/sources.html

@@ -19,6 +19,9 @@ <h1 class="truncate">Sources</h1>
   <div class="col m12 xl4 margin-bottom">
     <a href="{% url 'sync:validate-source' source_type='youtube-playlist' %}" class="btn">Add a YouTube playlist <i class="fab fa-youtube"></i></a>
   </div>
+  <div class="col m12 xl4 margin-bottom">
+    <a href="{% url 'sync:validate-source' source_type='twitch-channel' %}" class="btn">Add a Twitch channel <i class="fab fa-twitch"></i></a>
+  </div>
 </div>
 <div class="row no-margin-bottom">
   <div class="col s12">