Merge pull request #105 from xenova/dev

Various improvements

.github/workflows/python-package.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: ['3.6', '3.7', '3.8', '3.9', '3.10.0-beta.4']
 
     steps:
     - uses: actions/checkout@v2

README.rst

@@ -174,7 +174,7 @@ Supported sites:
 
 -  YouTube.com - Livestreams, past broadcasts and premieres.
 -  Twitch.tv - Livestreams, past broadcasts and clips.
--  Reddit.com - Livestreams (past broadcasts in development)
+-  Reddit.com - Livestreams, past broadcasts
 -  Facebook.com (currently in development) - Livestreams and past
    broadcasts.
 

chat_downloader/chat_downloader.py

@@ -25,8 +25,9 @@
 
 from .debugging import (
     log,
-    disable_logger,
-    set_log_level
+    set_testing_mode,
+    TestingModes,
+    TestingException
 )
 
 from .output.continuous_write import ContinuousWriter
@@ -43,7 +44,7 @@
     InvalidURL,
     ChatDownloaderError,
     ChatGeneratorError,
-    ParsingError,
+    ParsingError
 )
 
 
@@ -88,14 +89,6 @@ def get_chat(self, url=None,
                  inactivity_timeout=None,
                  max_messages=None,
 
-                 # Logging/debugging
-                 logging='none',
-                 pause_on_debug=False,
-                 exit_on_debug=False,
-                 testing=False,
-                 verbose=False,
-                 quiet=False,
-
                  message_groups=SiteDefault('message_groups'),
                  message_types=None,
 
@@ -144,22 +137,6 @@ def get_chat(self, url=None,
         :param max_messages: Maximum number of messages to retrieve, defaults
             to None (unlimited)
         :type max_messages: int, optional
-        :param logging: Level of logging to display, defaults to 'none'
-            ('info' if run from the CLI)
-        :type logging: str, optional
-        :param pause_on_debug: Pause on certain debug messages, defaults to False
-        :type pause_on_debug: bool, optional
-        :param exit_on_debug: Exit when something unexpected happens, defaults
-            to False
-        :type exit_on_debug: bool, optional
-        :param testing: Enable testing mode. This is equivalent to setting
-            logging to debug and enabling pause_on_debug. Defaults to False
-        :type testing: bool, optional
-        :param verbose: Print various debugging information. This is equivalent
-            to setting logging to debug. Defaults to False
-        :type verbose: bool, optional
-        :param quiet: Activate quiet mode (hide all output), defaults to False
-        :type quiet: bool, optional
         :param message_groups: List of messages groups (a predefined,
             site-specific collection of message types) to include
         :type message_groups: SiteDefault, optional
@@ -211,18 +188,6 @@ def get_chat(self, url=None,
         if not url:
             raise URLNotProvided('No URL provided.')
 
-        if testing:
-            logging = 'debug'
-            pause_on_debug = True
-
-        if verbose:
-            logging = 'debug'
-
-        if quiet or logging == 'none':
-            disable_logger()
-        else:
-            set_log_level(logging)
-
         original_params = locals()
         original_params.pop('self')
 
@@ -357,6 +322,12 @@ def run(propagate_interrupt=False, **kwargs):
     Create a single session and get the chat using the specified parameters.
     """
 
+    # Set testing mode
+    if kwargs.get('exit_on_debug'):
+        set_testing_mode(TestingModes.EXIT_ON_DEBUG)
+    elif kwargs.get('pause_on_debug'):
+        set_testing_mode(TestingModes.PAUSE_ON_DEBUG)
+
     init_param_names = get_default_args(ChatDownloader.__init__)
     program_param_names = get_default_args(ChatDownloader.get_chat)
 
@@ -394,15 +365,17 @@ def callback(item):
 
     except (
         ChatGeneratorError,
-        ParsingError
+        ParsingError,
+        TestingException
     ) as e:  # Errors which may be bugs
         log('error', '{}. Please report this at https://github.com/xenova/chat-downloader/issues/new/choose'.format(e))
 
     except ChatDownloaderError as e:  # Expected errors
         log('error', e)
 
     except ConnectionError as e:
-        log('error', 'Unable to establish a connection. Please check your internet connection. {}'.format(e))
+        log(
+            'error', 'Unable to establish a connection. Please check your internet connection. {}'.format(e))
 
     except RequestException as e:
         log('error', e)

chat_downloader/cli.py

@@ -21,6 +21,11 @@
     splitter
 )
 
+from .debugging import (
+    disable_logger,
+    set_log_level
+)
+
 
 def main():
 
@@ -55,7 +60,7 @@ def add_param(param_type, group, *keys, **kwargs):
 
         if is_boolean_flag:
             # If True by default, set action to store_false
-            # If False by default, set action to store_false
+            # If False by default, set action to store_true
 
             default = kwargs.pop('default', None)
             kwargs['action'] = 'store_{}'.format(
@@ -134,25 +139,25 @@ def add_init_param(group, *keys, **kwargs):
     add_chat_param(output_group, '--sort_keys', is_boolean_flag=True)
     add_chat_param(output_group, '--indent', type=lambda x: int_or_none(x, x))
 
+    # Debugging only available from the CLI
     debug_group = parser.add_argument_group('Debugging/Testing Arguments')
 
     on_debug_options = debug_group.add_mutually_exclusive_group()
-    add_chat_param(on_debug_options, '--pause_on_debug', is_boolean_flag=True)
-    add_chat_param(on_debug_options, '--exit_on_debug', is_boolean_flag=True)
+    on_debug_options.add_argument('--pause_on_debug', action='store_true',
+                                  help='Pause on certain debug messages, defaults to False')
+    on_debug_options.add_argument('--exit_on_debug', action='store_true',
+                                  help='Exit when something unexpected happens, defaults to False')
 
     debug_options = debug_group.add_mutually_exclusive_group()
+    debug_options.add_argument('--logging', choices=['none', 'debug', 'info', 'warning', 'error', 'critical'],
+                               help='Level of logging to display, defaults to info', default='info')
 
-    # overwrite default from method
-    get_chat_info['logging']['default'] = 'info'
-
-    add_chat_param(debug_options, '--logging',
-                   choices=['none', 'debug', 'info', 'warning', 'error', 'critical'])
-
-    add_chat_param(debug_options, '--testing', is_boolean_flag=True)
-    add_chat_param(debug_options, '--verbose', '-v', is_boolean_flag=True)
-    add_chat_param(debug_options, '--quiet', '-q', is_boolean_flag=True)
-
-    # TODO Add --do_not_print option
+    debug_options.add_argument('--testing', action='store_true',
+                               help='Enable testing mode. This is equivalent to setting logging to debug and enabling pause_on_debug. Defaults to False')
+    debug_options.add_argument('--verbose', '-v', action='store_true',
+                               help='Print various debugging information. This is equivalent to setting logging to debug. Defaults to False')
+    debug_options.add_argument('--quiet', '-q', action='store_true',
+                               help='Activate quiet mode (hide all output), defaults to False')
 
     # INIT PARAMS
     init_group = parser.add_argument_group('Initialisation Arguments')
@@ -166,5 +171,18 @@ def add_init_param(group, *keys, **kwargs):
 
     args = parser.parse_args()
 
+    # Modify debugging args:
+    if args.testing: # (only for CLI)
+        args.logging = 'debug'
+        args.pause_on_debug = True
+
+    if args.verbose:
+        args.logging = 'debug'
+
+    if args.quiet or args.logging == 'none':
+        disable_logger()
+    else:
+        set_log_level(args.logging)
+
     # Run with these arguments
     run(**args.__dict__)

chat_downloader/debugging.py

@@ -2,7 +2,64 @@
 
 import sys
 import os
-import chat_downloader
+
+from .metadata import __name__ as logger_name
+from .utils.core import pause
+
+from enum import Enum
+
+
+class TestingException(Exception):
+    """Raised when something unexpected happens while in testing mode"""
+
+
+class TestingModes(Enum):
+    # Currently unused
+    EXIT_ON_ERROR = 4
+    PAUSE_ON_ERROR = 3
+
+    # In use
+    EXIT_ON_DEBUG = 2
+    PAUSE_ON_DEBUG = 1
+    NONE = 0
+
+
+TESTING_MODE = TestingModes.NONE
+
+
+def set_testing_mode(new_mode):
+    global TESTING_MODE
+    TESTING_MODE = new_mode
+
+
+def log(level, items, to_pause=False, to_exit=False):
+    logger_at_level = getattr(logger, level, None)
+    if logger_at_level:
+        if not isinstance(items, (tuple, list)):
+            items = [items]
+        for item in items:
+            logger_at_level(item)
+
+        if to_exit and TESTING_MODE in (TestingModes.EXIT_ON_ERROR, TestingModes.EXIT_ON_DEBUG):
+            raise TestingException(
+                'Testing exception encountered, exiting program')
+
+        if to_pause and TESTING_MODE in (TestingModes.PAUSE_ON_ERROR, TestingModes.PAUSE_ON_DEBUG):
+            pause()
+
+
+def debug_log(*items):
+    """Method which simplifies the logging of debugging messages"""
+    log('debug', items, True, True)
+
+
+try:
+    import colorama
+    colorama.init()
+except (ImportError, OSError):
+    HAS_COLORAMA = False
+else:
+    HAS_COLORAMA = True
 
 
 def supports_colour():
@@ -36,18 +93,23 @@ def vt_codes_enabled_in_windows_registry():
     is_a_tty = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty()
 
     return is_a_tty and (
+        sys.platform != 'win32' or
+        HAS_COLORAMA or
+        'ANSICON' in os.environ or
+
         # Windows Terminal supports VT codes.
-        # Microsoft Visual Studio Code's built-in terminal supports colours.
+        'WT_SESSION' in os.environ or
 
-        (sys.platform != 'win32') or ('ANSICON' in os.environ) or ('WT_SESSION' in os.environ) or (
-            os.environ.get('TERM_PROGRAM') == 'vscode') or (vt_codes_enabled_in_windows_registry())
+        # Microsoft Visual Studio Code's built-in terminal supports colors.
+        os.environ.get('TERM_PROGRAM') == 'vscode' or
+        vt_codes_enabled_in_windows_registry()
     )
 
 
 if supports_colour():
-    import colorlog
-    handler = colorlog.StreamHandler()
-    handler.setFormatter(colorlog.ColoredFormatter(
+    import colorlog as log_module
+    handler = log_module.StreamHandler()
+    handler.setFormatter(log_module.ColoredFormatter(
         '[%(log_color)s%(levelname)s%(reset)s] %(message)s',
         log_colors={
             'DEBUG': 'cyan',
@@ -57,16 +119,13 @@ def vt_codes_enabled_in_windows_registry():
             'CRITICAL': 'bold_red',
         })
     )
-    log_module = colorlog
 
 else:  # fallback support
-    import logging
-    handler = logging.StreamHandler()
-    handler.setFormatter(logging.Formatter('[%(levelname)s] %(message)s'))
-    log_module = logging
+    import logging as log_module
+    handler = log_module.StreamHandler()
+    handler.setFormatter(log_module.Formatter('[%(levelname)s] %(message)s'))
 
 # Create logger object for this module
-logger_name = chat_downloader.__name__
 logger = log_module.getLogger(logger_name)
 
 # Define which loggers to display
@@ -83,15 +142,3 @@ def set_log_level(level):
 
 def disable_logger():
     logger.disabled = True
-
-
-def log(level, items, to_pause=False):
-    logger_at_level = getattr(logger, level, None)
-    if logger_at_level:
-        if not isinstance(items, (tuple, list)):
-            items = [items]
-        for item in items:
-            logger_at_level(item)
-
-        if to_pause:
-            input('Press Enter to continue...')

chat_downloader/metadata.py

@@ -7,4 +7,4 @@
 __email__ = 'admin@xenova.com'
 __copyright__ = '2020, 2021 xenova'
 __url__ = 'https://github.com/xenova/chat-downloader'
-__version__ = '0.1.4'
+__version__ = '0.1.5'

chat_downloader/sites/common.py

@@ -312,23 +312,6 @@ def _must_add_item(item, message_groups_dict, messages_groups_to_add, messages_t
 
         return item.get('message_type') in valid_message_types
 
-    @staticmethod
-    def _debug_log(params, *items):
-        """Method which simplifies the logging of debugging messages
-
-        :param params: Dictionary of parameters sent to the `get_chat` method
-        :type params: dict
-        :raises UnexpectedError: if something unexpected occurs, but is only
-            used when debugging
-        """
-        log(
-            'debug',
-            items,
-            params.get('pause_on_debug')
-        )
-        if params.get('exit_on_debug'):
-            raise UnexpectedError(items)
-
     def __init__(self,
                  **kwargs
                  ):