Merge pull request #107 from xenova/dev

Improved testing

Makefile

@@ -58,16 +58,20 @@ test-all: ## run tests on every Python version with tox
 	tox
 
 coverage: ## check code coverage quickly with the default Python
-	coverage run --source chat_downloader -m pytest
+	coverage run --source chat_downloader -m pytest -v
 	coverage report -m
 	coverage html
 	$(BROWSER) htmlcov/index.html
+	python docs/scripts/update_coverage.py
+
+coverage-report:
+	@coverage report --precision=2
 
 docs: ## generate Sphinx HTML documentation and README
 	$(MAKE) -C docs clean
 	$(MAKE) -C docs html
 	$(BROWSER) docs/_build/index.html
-	cd docs && python scripts/generate_readme.py > ../README.rst
+	python docs/scripts/generate_readme.py > README.rst
 
 servedocs: docs ## compile the docs watching for changes
 	watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D .

README.rst

@@ -10,18 +10,24 @@ Chat Downloader
 .. image:: https://img.shields.io/pypi/pyversions/chat-downloader
    :target: https://pypi.org/project/chat-downloader
    :alt: Python
+
 .. image:: https://img.shields.io/pypi/v/chat-downloader.svg
    :target: https://pypi.org/project/chat-downloader
    :alt: PyPI version
-.. image:: https://pepy.tech/badge/chat-downloader
+
+.. image:: https://pepy.tech/badge/chat-downloader/month
    :target: https://pypi.org/project/chat-downloader
    :alt: Downloads
+
 .. image:: https://img.shields.io/github/license/xenova/chat-downloader
   :target: https://github.com/xenova/chat-downloader/blob/master/LICENSE
   :alt: License
 
+.. image:: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/xenova/chat-downloader/master/docs/_dynamic/coverage.json
+  :target: https://pypi.org/project/chat-downloader
+  :alt: Coverage
+
 ..
-    [![PyPI Downloads](https://img.shields.io/pypi/dm/chat-downloader)](https://pypi.org/project/chat-downloader)
     [![GitHub issues](https://img.shields.io/github/issues/xenova/chat-downloader)](https://badge.fury.io/py/chat-downloader)
     [![GitHub forks](https://img.shields.io/github/forks/xenova/chat-downloader)](https://badge.fury.io/py/chat-downloader)
     [![GitHub stars](https://img.shields.io/github/stars/xenova/chat-downloader)](https://badge.fury.io/py/chat-downloader)

chat_downloader/chat_downloader.py

@@ -258,15 +258,8 @@ def log_on_inactivity_timeout():
                     x, format_name=params['format'])
 
                 if params['output']:
-                    output_file = ContinuousWriter(
-                        params['output'], indent=params['indent'], sort_keys=params['sort_keys'], overwrite=params['overwrite'])
-
-                    if output_file.is_default():
-                        chat.callback = lambda item: output_file.write(
-                            chat.format(item), flush=True)
-                    else:
-                        chat.callback = lambda item: output_file.write(
-                            item, flush=True)
+                    chat.attach_writer(ContinuousWriter(
+                        params['output'], indent=params['indent'], sort_keys=params['sort_keys'], overwrite=params['overwrite']))
 
                 chat.site = site_object
 
@@ -355,7 +348,7 @@ def callback(item):
                 pass
         else:
             def callback(item):
-                safe_print(chat.format(item), flush=True)
+                chat.print_formatted(item)
 
         for message in chat:
             callback(message)

chat_downloader/cli.py

@@ -27,7 +27,7 @@
 )
 
 
-def main():
+def main(cli_args=None):
 
     parser = argparse.ArgumentParser(
         description=__summary__,
@@ -169,10 +169,10 @@ def add_init_param(group, *keys, **kwargs):
     parser._positionals.title = 'Mandatory Arguments'
     parser._optionals.title = 'General Arguments'
 
-    args = parser.parse_args()
+    args = parser.parse_args(args=cli_args)
 
     # Modify debugging args:
-    if args.testing: # (only for CLI)
+    if args.testing:  # (only for CLI)
         args.logging = 'debug'
         args.pause_on_debug = True
 

chat_downloader/sites/common.py

@@ -9,7 +9,10 @@
     InvalidParameter,
     RetriesExceeded,
     CookieError,
-    UnexpectedError
+    UnexpectedError,
+    URLNotProvided,
+    SiteNotSupported,
+    InvalidURL
 )
 
 from ..utils.core import (
@@ -223,6 +226,9 @@ def __init__(self, chat=None, callback=None, title=None, duration=None, is_live=
         # TODO
         # author/user/uploader/creator
 
+        self._output_writer = None
+        self._output_callback = None
+
     def __iter__(self):
         """Allows the object to be iterable
 
@@ -231,25 +237,44 @@ def __iter__(self):
         """
         return self
 
+    def attach_writer(self, writer):
+        # writer is a ContinuousWriter
+        self._output_writer = writer
+        if self._output_writer.is_default():
+            self._output_callback = lambda item: self._output_writer.write(
+                self.format(item), flush=True)
+        else:
+            self._output_callback = lambda item: self._output_writer.write(
+                item, flush=True)
+
     def __next__(self):
         """Get the next chat message from the generator
 
         :return: The next chat item
         :rtype: dict
         """
-        item = next(self.chat)
-        if self.callback:
-            self.callback(item)
-        return item
-
-    def print_formatted(self, item):
+        try:
+            item = next(self.chat)
+            if self.callback:  # user-defined callback
+                self.callback(item)
+
+            if self._output_callback:  # output callback
+                self._output_callback(item)
+
+            return item
+        except StopIteration as e:
+            # Safely close output file when done
+            if self._output_writer is not None:
+                self._output_writer.close()
+            raise e
+
+    def print_formatted(self, item, flush=True):
         """Safely print the formatted message
 
         :param item: The chat item to be printed
         :type item: dict
         """
-        formatted = self.format(item)
-        safe_print(formatted)
+        safe_print(self.format(item), flush=flush)
 
     def format(self, item):
         """Format chat messages
@@ -278,6 +303,14 @@ class BaseChatDownloader:
 
     # For general tests (non-site specific)
     _TESTS = [
+        {
+            'name': 'Inactivity timeout',
+            'params': {
+                'url': 'https://twitch.tv/xenova',
+                'inactivity_timeout': 5,
+                'timeout': 20,  # As a fallback
+            }
+        },
         {
             'name': 'Get a certain number of messages from a livestream.',
             'params': {
@@ -289,6 +322,48 @@ class BaseChatDownloader:
             'expected_result': {
                 'messages_condition': lambda messages: len(messages) <= 10,
             }
+        },
+
+
+        {
+            'name': 'Scheme not supplied',
+            'params': {
+                'url': 'www.youtube.com/watch?v=5qap5aO4i9A',
+                'max_messages': 10,
+                'timeout': 60,  # As a fallback
+            },
+            'expected_result': {
+                'messages_condition': lambda messages: len(messages) <= 10,
+            }
+        },
+
+        # Tests for errors
+        {
+            'name': 'No URL provided.',
+            'params': {
+                'url': '',
+            },
+            'expected_result': {
+                'error': URLNotProvided
+            }
+        },
+        {
+            'name': 'Site not supported',
+            'params': {
+                'url': 'https://www.example.com',
+            },
+            'expected_result': {
+                'error': SiteNotSupported
+            }
+        },
+        {
+            'name': 'Invalid URL',
+            'params': {
+                'url': '#',
+            },
+            'expected_result': {
+                'error': InvalidURL
+            }
         }
     ]
 
@@ -530,11 +605,6 @@ def retry(attempt_number, max_attempts, error=None, retry_timeout=None, text=Non
         if isinstance(error, Exception):
             retry_text += ' {} ({})'.format(error, error.__class__.__name__)
 
-        log(
-            'warning',
-            text + [retry_text]
-        )
-
         if isinstance(error, JSONDecodeError):
             log(
                 'debug',
@@ -544,6 +614,11 @@ def retry(attempt_number, max_attempts, error=None, retry_timeout=None, text=Non
             if page_title:
                 log('debug', 'Title: {}'.format(page_title))
 
+        log(
+            'warning',
+            text + [retry_text]
+        )
+
         if must_sleep:
             timed_input(time_to_sleep)
         else: