Consistent code formatting using black, isort (#17)

.github/workflows/build.yml

@@ -25,13 +25,15 @@ jobs:
         python -m pip install --upgrade pip
         pip install -e .
         pip install -r requirements-test.txt
-    - name: Lint with flake8
+    - name: Lint with flake8 and black
       run: |
-        pip install flake8
+        pip install flake8 black isort
         # stop the build if there are Python syntax errors or undefined names
         flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+        # check using isort
+        isort --check-only --project popmon --multi-line=3 --trailing-comma --force-grid-wrap=0 --use-parentheses --line-width=88
+        # check using black
+        black --check .
     - name: Test with pytest
       run: |
         pip install pytest

Makefile

@@ -0,0 +1,3 @@
+lint:
+	isort --project popmon --multi-line=3 --trailing-comma --force-grid-wrap=0 --use-parentheses --line-width=88
+	black .

docs/source/conf.py

@@ -5,10 +5,11 @@
 #
 
 import os
-# from unittest.mock import MagicMock
 
 import popmon
 
+# from unittest.mock import MagicMock
+
 
 # Classes that use non-python modules are not always available in the
 # RTD environment. By mocking them we can still import these classes
@@ -33,64 +34,61 @@
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = [
-    'sphinx.ext.autodoc',
-    'sphinx.ext.mathjax',
-    'sphinx.ext.ifconfig',
-]
+extensions = ["sphinx.ext.autodoc", "sphinx.ext.mathjax", "sphinx.ext.ifconfig"]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
 # source_suffix = ['.rst', '.md']
-source_suffix = '.rst'
+source_suffix = ".rst"
 
 # The encoding of source files.
 # source_encoding = 'utf-8-sig'
 
 # The master toctree document.
-master_doc = 'index'
+master_doc = "index"
 
 # General information about the project.
-project = 'Population Shift Monitoring'
-copyright = '2020, ING Bank N.V.'
-author = 'ING Wholesale Banking Advanced Analytics'
+project = "Population Shift Monitoring"
+copyright = "2020, ING Bank N.V."
+author = "ING Wholesale Banking Advanced Analytics"
 version = popmon.version.full_version
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = 'en'
+language = "en"
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
-exclude_patterns = ['*test*', 'popmon.tutorials.*', 'popmon.decorators.*']
+exclude_patterns = ["*test*", "popmon.tutorials.*", "popmon.decorators.*"]
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = "sphinx"
 
 # If true, `todo` and `todoList` produce output, else they produce nothing.
 todo_include_todos = False
 
 # -- Options for HTML output ----------------------------------------------
 
 # on_rtd is whether we are on readthedocs.org, this line of code grabbed from docs.readthedocs.org
-on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
+on_rtd = os.environ.get("READTHEDOCS", None) == "True"
 
 if not on_rtd:
     import sphinx_rtd_theme
+
     html_theme = "sphinx_rtd_theme"
     html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 # otherwise, readthedocs.org uses their theme by default, so no need to specify it
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["_static"]
 
 # If false, no index is generated.
 html_use_index = True
@@ -111,23 +109,20 @@
 # Sphinx supports the following languages:
 #   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
 #   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
-html_search_language = 'en'
+html_search_language = "en"
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'POPMONdoc'
+htmlhelp_basename = "POPMONdoc"
 
 # -- Options for LaTeX output ---------------------------------------------
 
 latex_elements = {
     # The paper size ('letterpaper' or 'a4paper').
     # 'papersize': 'letterpaper',
-
     # The font size ('10pt', '11pt' or '12pt').
     # 'pointsize': '10pt',
-
     # Additional stuff for the LaTeX preamble.
     # 'preamble': '',
-
     # Latex figure (float) alignment
     # 'figure_align': 'htbp',
 }
@@ -136,28 +131,36 @@
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, 'popmon.tex', 'POPMON Documentation',
-     'ING Wholesale Banking Advanced Analytics', 'manual'),
+    (
+        master_doc,
+        "popmon.tex",
+        "POPMON Documentation",
+        "ING Wholesale Banking Advanced Analytics",
+        "manual",
+    )
 ]
 
 # -- Options for manual page output ---------------------------------------
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [
-    (master_doc, 'popmon', 'POPMON Documentation',
-     [author], 1)
-]
+man_pages = [(master_doc, "popmon", "POPMON Documentation", [author], 1)]
 
 # -- Options for Texinfo output -------------------------------------------
 
 # Grouping the document tree into Texinfo files. List of tuples
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    (master_doc, 'popmon', 'POPMON Documentation',
-     author, 'popmon', 'Population Shift Monitoring.',
-     'Miscellaneous'),
+    (
+        master_doc,
+        "popmon",
+        "POPMON Documentation",
+        author,
+        "popmon",
+        "Population Shift Monitoring.",
+        "Miscellaneous",
+    )
 ]
 
 

popmon/__init__.py

@@ -1,11 +1,10 @@
 # flake8: noqa
-from .version import version as __version__
-
 # pandas/spark dataframe decorators
 from popmon import decorators
 
 # histogram and report functions
-from .hist.filling import make_histograms, get_time_axes, get_bin_specs
+from .hist.filling import get_bin_specs, get_time_axes, make_histograms
+from .pipeline.metrics import df_stability_metrics, stability_metrics
+from .pipeline.report import df_stability_report, stability_report
 from .stitching import stitch_histograms
-from .pipeline.report import stability_report, df_stability_report
-from .pipeline.metrics import stability_metrics, df_stability_metrics
+from .version import version as __version__

popmon/alerting/__init__.py

@@ -1,16 +1,23 @@
-from ..alerting.compute_tl_bounds import ComputeTLBounds, collect_traffic_light_bounds, \
-    traffic_light, pull_bounds, DynamicBounds, TrafficLightAlerts, traffic_light_summary, StaticBounds
 from ..alerting.alerts_summary import AlertsSummary
-
+from ..alerting.compute_tl_bounds import (
+    ComputeTLBounds,
+    DynamicBounds,
+    StaticBounds,
+    TrafficLightAlerts,
+    collect_traffic_light_bounds,
+    pull_bounds,
+    traffic_light,
+    traffic_light_summary,
+)
 
 __all__ = [
-    'ComputeTLBounds',
-    'collect_traffic_light_bounds',
-    'traffic_light',
-    'pull_bounds',
-    'DynamicBounds',
-    'TrafficLightAlerts',
-    'traffic_light_summary',
-    'StaticBounds',
-    'AlertsSummary',
+    "ComputeTLBounds",
+    "collect_traffic_light_bounds",
+    "traffic_light",
+    "pull_bounds",
+    "DynamicBounds",
+    "TrafficLightAlerts",
+    "traffic_light_summary",
+    "StaticBounds",
+    "AlertsSummary",
 ]

popmon/alerting/alerts_summary.py

@@ -1,15 +1,25 @@
+import fnmatch
+
 import numpy as np
 import pandas as pd
+
 from ..base import Module
-import fnmatch
 
 
 class AlertsSummary(Module):
     """The module AlertsSummary combines the alerts-summaries of all individual features
 
     It combines the alerts-summaries of all individual features into an artificial feature "_AGGREGATE_".
     """
-    def __init__(self, read_key, store_key='', features=None, ignore_features=None, combined_variable='_AGGREGATE_'):
+
+    def __init__(
+        self,
+        read_key,
+        store_key="",
+        features=None,
+        ignore_features=None,
+        combined_variable="_AGGREGATE_",
+    ):
         """Initialize an instance of AlertsSummary module.
 
         :param str read_key: key of input data to read from datastore.
@@ -36,35 +46,41 @@ def transform(self, datastore):
         if len(features) == 0:
             return datastore
 
-        self.logger.info(f'Combining alerts into artificial variable \"{self.combined_variable}\"')
+        self.logger.info(
+            f'Combining alerts into artificial variable "{self.combined_variable}"'
+        )
 
         # STEP 1: loop over features where alerts exist
         df_list = []
         for feature in features:
             # basic checks if feature object is filled correctly
-            df = (self.get_datastore_object(data, feature, dtype=pd.DataFrame)).copy(deep=False)
-            df.columns = [feature + '_' + c for c in df.columns]
+            df = (self.get_datastore_object(data, feature, dtype=pd.DataFrame)).copy(
+                deep=False
+            )
+            df.columns = [feature + "_" + c for c in df.columns]
             df_list.append(df)
 
         # the different features could technically have different indices.
         # will only merge alerts if all indices are the same
         if len(df_list) >= 2:
             for df in df_list[1:]:
                 if not np.array_equal(df_list[0].index, df.index):
-                    self.logger.warning('indices of features are different. no alerts summary generated.')
+                    self.logger.warning(
+                        "indices of features are different. no alerts summary generated."
+                    )
                     return datastore
 
         # STEP 2: Concatenate the dataframes, there was one for each original feature.
         tlv = pd.concat(df_list, axis=1)
         dfc = pd.DataFrame(index=tlv.index)
 
         # worst traffic light
-        cols = fnmatch.filter(tlv.columns, '*_worst')
-        dfc['worst'] = tlv[cols].values.max(axis=1) if len(cols) else 0
+        cols = fnmatch.filter(tlv.columns, "*_worst")
+        dfc["worst"] = tlv[cols].values.max(axis=1) if len(cols) else 0
         # colors of traffic lights
         for color in ["green", "yellow", "red"]:
-            cols = fnmatch.filter(tlv.columns, '*_n_{}'.format(color))
-            dfc['n_{}'.format(color)] = tlv[cols].values.sum(axis=1) if len(cols) else 0
+            cols = fnmatch.filter(tlv.columns, "*_n_{}".format(color))
+            dfc["n_{}".format(color)] = tlv[cols].values.sum(axis=1) if len(cols) else 0
 
         # store combination of traffic alerts
         data[self.combined_variable] = dfc