Merge pull request #101 from xenova/youtube-member-fix

Added reddit.com extractor and added fix for upcoming member-only streams

README.rst

@@ -174,6 +174,7 @@ Supported sites:
 
 -  YouTube.com - Livestreams, past broadcasts and premieres.
 -  Twitch.tv - Livestreams, past broadcasts and clips.
+-  Reddit.com - Livestreams (past broadcasts in development)
 -  Facebook.com (currently in development) - Livestreams and past
    broadcasts.
 

chat_downloader/chat_downloader.py

@@ -15,13 +15,14 @@
 from .sites import get_all_sites
 
 from .formatting.format import ItemFormatter
-from .utils import (
+from .utils.core import (
     safe_print,
     get_default_args,
-    update_dict_without_overwrite,
-    TimedGenerator
+    update_dict_without_overwrite
 )
 
+from .utils.timed_utils import TimedGenerator
+
 from .debugging import (
     log,
     disable_logger,
@@ -49,7 +50,9 @@
     NoContinuation,
     UserNotFound,
     ChatGeneratorError,
-    ParsingError
+    ParsingError,
+    SiteError,
+    NoVideos
 )
 
 
@@ -257,6 +260,7 @@ def get_chat(self, url=None,
                 if not get_chat:
                     raise NotImplementedError(
                         '{} has not been implemented in {}.'.format(function_name, site.__name__))
+
                 chat = get_chat(match, params)
                 log('debug', 'Match found: "{}". Running "{}" function in "{}".'.format(
                     match, function_name, site.__name__))
@@ -297,12 +301,8 @@ def log_on_inactivity_timeout():
                     output_file = ContinuousWriter(
                         output, indent=indent, sort_keys=sort_keys, overwrite=overwrite)
 
-                    if output_file.is_default():
-                        chat.callback = lambda x: output_file.write(
-                            chat.format(x), flush=True)
-                    else:
-                        chat.callback = lambda x: output_file.write(
-                            x, flush=True)
+                    chat.callback = lambda item: output_file.write(
+                        chat.format(item) if output_file.is_default() else item, flush=True)
 
                 chat.site = site_object
 
@@ -404,7 +404,9 @@ def callback(item):
         InvalidURL,
         RetriesExceeded,
         NoContinuation,
-        UserNotFound
+        UserNotFound,
+        SiteError,
+        NoVideos
     ) as e:  # Expected errors
         log('error', e)
     except (

chat_downloader/cli.py

@@ -15,7 +15,7 @@
     __program__
 )
 
-from .utils import (
+from .utils.core import (
     get_default_args,
     int_or_none
 )

chat_downloader/errors.py

@@ -26,10 +26,17 @@ class VideoNotFound(ChatDownloaderError):
     """Raised when video cannot be found."""
     pass
 
+
 class UserNotFound(ChatDownloaderError):
     """Raised when user cannot be found."""
     pass
 
+
+class NoVideos(ChatDownloaderError):
+    """Raised when a channel does not have any videos."""
+    pass
+
+
 class ParsingError(ChatDownloaderError):
     """Raised when video data cannot be parsed."""
     pass
@@ -55,6 +62,11 @@ class NoChatReplay(ChatDownloaderError):
     pass
 
 
+class ChatDisabled(ChatDownloaderError):
+    """Raised when the chat is disabled."""
+    pass
+
+
 class URLNotProvided(ChatDownloaderError):
     """Raised when no url is provided."""
     pass
@@ -75,8 +87,8 @@ class SiteNotSupported(ChatDownloaderError):
     pass
 
 
-class TwitchError(ChatDownloaderError):
-    """Raised when an error occurs with a Twitch video."""
+class SiteError(ChatDownloaderError):
+    """Raised when an error occurs with a specific site."""
     pass
 
 

chat_downloader/formatting/format.py

@@ -2,7 +2,7 @@
 import os
 import json
 
-from ..utils import (
+from ..utils.core import (
     nested_update,
     multi_get,
     microseconds_to_timestamp

chat_downloader/metadata.py

@@ -7,4 +7,4 @@
 __email__ = 'admin@xenova.com'
 __copyright__ = '2020, 2021 xenova'
 __url__ = 'https://github.com/xenova/chat-downloader'
-__version__ = '0.1.2'
+__version__ = '0.1.3'

chat_downloader/output/continuous_write.py

@@ -3,7 +3,7 @@
 import csv
 import shutil
 
-from ..utils import flatten_json
+from ..utils.core import flatten_json
 
 
 class CW:

chat_downloader/sites/__init__.py

@@ -3,6 +3,7 @@
 from .youtube import YouTubeChatDownloader
 from .twitch import TwitchChatDownloader
 from .facebook import FacebookChatDownloader
+from .reddit import RedditChatDownloader
 from .common import BaseChatDownloader
 
 

chat_downloader/sites/common.py

@@ -12,13 +12,13 @@
     UnexpectedError
 )
 
-from ..utils import (
+from ..utils.core import (
     get_title_of_webpage,
     pause,
-    timed_input,
     safe_print
 )
 
+from ..utils.timed_utils import timed_input
 from ..debugging import log
 
 
@@ -213,8 +213,8 @@ def __init__(self, chat=None, callback=None, title=None, duration=None, is_live=
         """
 
         self.chat = chat
-
         self.callback = callback
+
         self.title = title
         self.duration = duration
         self.is_live = is_live
@@ -463,14 +463,6 @@ def matches(cls, url):
 
         return None
 
-    def get_chat(self, **kwargs):
-        """This method should be implemented in a subclass and should return
-        the appropriate `Chat` object with respect to the specified parameters.
-
-        :raises NotImplementedError: if not implemented and called from a subclass
-        """
-        raise NotImplementedError
-
     def generate_urls(self, **kwargs):
         """This method should be implemented in a subclass and should return
         a generator which yields URLs for testing.
@@ -508,15 +500,15 @@ def _move_to_dict(info, dict_name, replace_key=None, create_when_empty=False, *i
         return new_dict
 
     @staticmethod
-    def retry(attempt_number, max_attempts, error, retry_timeout=None, text=None):
+    def retry(attempt_number, max_attempts, error=None, retry_timeout=None, text=None):
         """Retry to occur after an error occurs
 
         :param attempt_number: The current attempt number
         :type attempt_number: int
         :param max_attempts: The maximum number of attempts allowed
         :type max_attempts: int
-        :param error: The error which was raised
-        :type error: Exception
+        :param error: The error which was raised, defaults to None
+        :type error: Exception, optional
         :param retry_timeout: The number of seconds to sleep after failing,
             defaults to None (i.e. use exponential back-off)
         :type retry_timeout: float, optional
@@ -550,8 +542,10 @@ def retry(attempt_number, max_attempts, error, retry_timeout=None, text=None):
         else:
             sleep_text = ''
 
-        retry_text = 'Retry #{} {}. {} ({})'.format(
-            attempt_number, sleep_text, error, error.__class__.__name__)
+        retry_text = 'Retry #{} {}.'.format(attempt_number, sleep_text)
+
+        if isinstance(error, Exception):
+            retry_text += ' {} ({})'.format(error, error.__class__.__name__)
 
         log(
             'warning',
@@ -568,7 +562,6 @@ def retry(attempt_number, max_attempts, error, retry_timeout=None, text=None):
                 log('debug', 'Title: {}'.format(page_title))
 
         if must_sleep:
-            # time.sleep(time_to_sleep)
             timed_input(time_to_sleep)
         else:
             pause()

chat_downloader/sites/facebook.py

@@ -4,7 +4,7 @@
     Remapper as r,
     Image
 )
-from ..utils import (
+from ..utils.core import (
     remove_prefixes,
     multi_get,
     try_get_first_value,