Add Formatters to coverage (#133)

Bring formatters to 100% test coverage.

Drive-by:
* Remove GSheet formatter
* Remove unused functions and dependencies

.coveragerc

@@ -4,7 +4,6 @@ omit =
   scrapd/cli/*
   scrapd/config.py
   scrapd/core/constant.py
-  scrapd/core/formatter.py
   scrapd/core/version.py
   scrapd/main.py
   scrapd/tests/*

CHANGELOG.md

@@ -13,8 +13,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
-- Refactor the functions parsing the fields to simplify their maintenance.
-- Remove LXML as a dependency. [#]
+- Refactor the functions parsing the fields to simplify their maintenance and improve the quality of the parsing.
+- Remove LXML as a dependency. [#117]
+- Dates are stored internaly as date objects instead of strings. The formatting is delegated to the `Formatters`
+  themselves. [#125]
+- Remove GSheet support. [#]
+
+### Fixed
+
+- Fix parsing of fatalities without date of birth. [#125]
 
 ## [[1.5.1]] - 2019-04-25
 
@@ -131,4 +138,5 @@ This first version allows a user to retrieve traffic fatality repports for a cer
 [#85]: https://github.com/scrapd/scrapd/pull/85
 [#90]: https://github.com/scrapd/scrapd/pull/90
 [#116]: https://github.com/scrapd/scrapd/pull/116
-[#]: https://github.com/scrapd/scrapd/pull/
+[#117]: https://github.com/scrapd/scrapd/pull/117
+[#125]: https://github.com/scrapd/scrapd/pull/125

README.rst

@@ -154,15 +154,6 @@ of the process::
     }
   ]
 
-
-Export the results to Google Sheets::
-
-  scrapd -v \
-    --from "Feb 1 2019" \
-    --format gsheets \
-    --gcredentials creds.json \
-    --gcontributors "remy.greinhofer@gmail.com:user:writer"
-
 Speed and accuracy
 ------------------
 

requirements.txt

@@ -1,16 +1,7 @@
 aiohttp==3.5.4
-anyconfig==0.9.9
 click==7.0
 dateparser==0.7.1
-google-api-python-client==1.7.8
-google-auth-httplib2==0.0.3
-google-auth-oauthlib==0.2.0
-gspread==3.1.0
 loguru==0.2.5
-pbr==5.2.0
-tabulate==0.8.3
+pbr>=5.2.0
 tenacity==5.0.4
 
-# TODO(remyg): These libraries are deprecated. Use a modern authentication method and remove them.
-oauth2client
-PyOpenSSL

scrapd/cli/base.py

@@ -3,7 +3,6 @@
 import sys
 
 from loguru import logger
-from tabulate import tabulate
 
 
 class AbstractCommand:
@@ -40,33 +39,3 @@ def execute(self):
     def _execute(self):
         """Define the internal execution of the command."""
         raise NotImplementedError
-
-    def display_tabular_data(self):
-        """
-        Show the results in a table on the screen.
-
-        If not header is defined, only the data is displayed, otherwise, the results will be shown in a table.
-        """
-        # Nothing to display if there is no data.
-        if not self.data:
-            return
-
-        # Define the table format based on the headers content.
-        table_format = "simple" if self.headers else "plain"
-
-        # Print the results.
-        print(tabulate(self.data, self.headers, tablefmt=table_format))
-
-    def placeholder_dict(self):
-        """
-        Convert the set of placeholders to a dictionary.
-
-        Placeholders are a set of key/value pairs. Their format is `key=value`. During the convertion, the `key`
-        becomes the key of the entry in the dictionary, the `value` becomes the value of the entry.
-
-        :param set placeholder: the placeholder to convert
-        :return dict: a dictionary representing the placeholder set.
-        """
-        placeholders = self.args.get('placeholder', [])
-        kvp_list = [kvp.split('=', 1) for kvp in placeholders if '=' in kvp]
-        return dict(kvp_list)

scrapd/cli/cli.py

@@ -1,13 +1,11 @@
 """Define the top-level cli command."""
 import asyncio
 import logging
-import os
 import sys
 
 import click
 from loguru import logger
 
-from scrapd import config
 from scrapd.cli.base import AbstractCommand
 from scrapd.core import apd
 from scrapd.core.formatter import Formatter
@@ -45,11 +43,6 @@ def cli(ctx, format_, from_, gcontributors, gcredentials, pages, to, verbose):
     ctx.obj = {**ctx.params}
     ctx.auto_envvar_prefix = 'VZ'
 
-    # Load defaults from configuration file if any.
-    cfg_path = os.path.join(click.get_app_dir(APP_NAME), APP_NAME + '.conf')
-    cfg = cfg_path if os.path.exists(cfg_path) else None
-    ctx.default_map = config.load(cfg, with_defaults=True, validate=True)
-
     # Configure logger.
     INITIAL_LOG_LEVEL = logging.WARNING
     LOG_FORMAT_COMPACT = "<level>{message}</level>"
@@ -81,9 +74,6 @@ class Retrieve(AbstractCommand):
 
     def _execute(self):
         """Define the internal execution of the command."""
-        # Check the params.
-        self.check_params()
-
         # Collect the results.
         results, _ = asyncio.run(apd.async_retrieve(
             self.args['pages'],
@@ -97,15 +87,3 @@ def _execute(self):
         format_ = self.args['format_'].lower()
         formatter = Formatter(format_)
         formatter.print(results)
-
-    def check_params(self):
-        """
-        Raise an exception if a custom parameters is invalid or missing.
-
-        This is to avoid collecting all the data then failing due to a missing parameter.
-        """
-        if self.args['format_'].lower() == 'gsheets':
-            if not self.args.get('gcredentials'):
-                raise click.ClickException('Google credentials are required.')
-            if not self.args.get('gcontributors'):
-                raise click.ClickException('At least 1 contributor is required.')

scrapd/core/formatter.py

@@ -12,7 +12,6 @@
 import sys
 
 from scrapd.core.constant import Fields
-from scrapd.core.gsheets import GSheets
 
 CSVFIELDS = [
     Fields.CRASHES,
@@ -54,12 +53,15 @@ def _get_formatter(self):
         formatter = self.formatters.get(self.format, self)
         return formatter()
 
-    def print(self, results, **kwargs):
+    def print(self, results, **kwargs):  # pragma: no cover
         """
         Print the results with the appropriate formatter.
 
         :param list(dict) results: the results to display.
         """
+        # Coverage is disabled for this function as the fact that we set output to sys.stdout early messes up
+        # with capsys in pytest, and prevent it to capture the out correctly.
+        # https://github.com/pytest-dev/pytest/issues/1132
         formatter = self._get_formatter()
         formatter.printer(results, **kwargs)
 
@@ -69,7 +71,6 @@ def date_serialize(self, obj):
 
         :rtype: str
         """
-
         if isinstance(obj, (datetime.datetime, datetime.date)):
             return obj.strftime("%m/%d/%Y")
         raise TypeError("Type %s not serializable" % type(obj))
@@ -91,7 +92,6 @@ def to_json_string(self, results):
 
         :rtype: str
         """
-
         return json.dumps(results, sort_keys=True, indent=2, default=self.date_serialize)
 
 
@@ -140,28 +140,6 @@ def printer(self, results, **kwargs):  # noqa: D102
         writer.writerows(results)
 
 
-class GSheetFormatter(Formatter):
-    """
-    Define the GSheet formatter.
-
-    Stores the results into a Google Sheets document.
-    """
-
-    __format_name__ = 'gsheets'
-
-    def printer(self, results, **kwargs):  # noqa: D102
-        credentials = kwargs.get('gcredentials')
-        if not credentials:
-            raise AttributeError('Google credentials are required.')
-        contributors = kwargs.get('gcontributors', '').split(',')
-        if not contributors:
-            raise AttributeError('At least 1 contributor is required.')
-        gs = GSheets(credentials, contributors)
-        gs.authenticate()
-        gs.create(datetime.datetime.now().strftime('%Y-%m-%d'))
-        gs.add_csv_data(CSVFIELDS, results)
-
-
 class CountFormatter(Formatter):
     """
     Define the Count formatter.