Merge pull request #133 from ing-bank/develop

v0.4.1

.github/dependabot.yml

@@ -0,0 +1,18 @@
+version: 2
+updates:
+- package-ecosystem: pip
+  directory: "/"
+  schedule:
+    interval: daily
+    time: "04:00"
+  open-pull-requests-limit: 10
+  target-branch: develop
+  allow:
+  - dependency-type: direct
+  - dependency-type: indirect
+  ignore:
+  - dependency-name: histogrammar
+    versions:
+    - 1.0.20
+    - 1.0.21
+    - 1.0.23

.github/workflows/build.yml

@@ -1,6 +1,3 @@
-# This workflow will install Python dependencies, run tests and lint with a single version of Python
-# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
-
 name: build
 
 on:
@@ -10,15 +7,19 @@ on:
     branches: [ master, develop ]
 
 jobs:
-  build:
-    runs-on: ubuntu-latest
+  test:
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python: [3.6, 3.7, 3.8]
+    runs-on: ${{ matrix.os }}
 
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Python 3.6
+    - name: Set up Python ${{ matrix.python }}
       uses: actions/setup-python@v1
       with:
-        python-version: 3.6
+        python-version: ${{ matrix.python }}
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
@@ -30,6 +31,25 @@ jobs:
     - name: Test with pytest
       run: |
         pytest -m "not spark"
+        
+  test_spark:
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python: [3.6]
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e .
+        pip install -r requirements-test.txt
     - name: Install Spark
       env:
         BUILD_DIR: "/home/runner/work/" #${{ github.workspace }}
@@ -39,6 +59,7 @@ jobs:
         SPARK_HOME: "/home/runner/work/spark/" #${{ github.workspace }}/spark/
         SPARK_LOCAL_IP: "localhost"
       run: |
+        sudo apt-get update
         sudo apt-get -y install openjdk-8-jdk
         curl https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz --output ${BUILD_DIR}/spark.tgz
         tar -xvzf ${BUILD_DIR}/spark.tgz && mv spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION} ${SPARK_HOME}
@@ -51,7 +72,9 @@ jobs:
 
   examples:
     runs-on: ubuntu-latest
-    needs: build
+    needs: 
+    - test
+    - test_spark
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python 3.6
@@ -62,18 +85,16 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install -e .
-      - name:
+      - name: Build examples
         run: |
           cd examples
           python synthetic_data.py
           python flight_delays.py
-
       - uses: actions/upload-artifact@v2
         with:
           name: synthetic-report
           path: examples/test_data_report.html
           if-no-files-found: error
-
       - uses: actions/upload-artifact@v2
         with:
           name: flight-delays-report

.pre-commit-config.yaml

@@ -1,21 +1,26 @@
 repos:
 -   repo: https://github.com/psf/black
-    rev: 20.8b1
+    rev: 21.6b0
     hooks:
     - id: black
 -   repo: https://github.com/pycqa/isort
-    rev: 5.8.0
+    rev: 5.9.1
     hooks:
       - id: isort
         files: '.*'
-        args: [ --profile=black, --project=popmon, --thirdparty histogrammar, --thirdparty pybase64 ]
--   repo: https://gitlab.com/pycqa/flake8
-    rev: "3.9.0"
+        args: [ --profile=black, --project=popmon ]
+-   repo: https://github.com/PyCQA/flake8
+    rev: "3.9.2"
     hooks:
     -   id: flake8
-        args: [ "--select=E9,F63,F7,F82"]
+        additional_dependencies:
+            - flake8-comprehensions
+        args: [ "--select=E9,F63,F7,F82,C4"]
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.12.0
+    rev: v2.19.4
     hooks:
     -   id: pyupgrade
-        args: ['--py36-plus','--exit-zero-even-if-changed']
+        args: ['--py36-plus','--exit-zero-even-if-changed']
+
+ci:
+    autoupdate_commit_msg: 'ci: pre-commit-config update'

.versionrc.json

@@ -0,0 +1,16 @@
+{
+    "header": "Release notes",
+    "types": [
+      {"type": "feat", "section": "\uD83C\uDF89 Features"},
+      {"type": "fix", "section": "\uD83D\uDC1B Bug fixes"},
+      {"type": "perf", "section": "⚡ Performance improvements"},
+      {"type": "docs", "section": "\uD83D\uDCD6 Documentation"},
+      {"type": "style", "section": "\uD83D\uDC77\u200D♂️ Internal improvements"},
+      {"type": "refactor", "section": "\uD83D\uDC77\u200D♂️ Internal improvements"},
+      {"type": "build", "section": "⬆️ Dependencies"},
+      {"type": "chore", "hidden": true},
+      {"type": "test", "hidden": true}
+    ],
+    "commitUrlFormat": "https://github.com/ing-bank/popmon/commits/{{hash}}",
+    "compareUrlFormat": "https://github.com/ing-bank/popmon/compare/{{previousTag}}...{{currentTag}}"
+}

CHANGELOG.md

@@ -0,0 +1,35 @@
+# Release notes
+## [Version 0.4.1](https://github.com/ing-bank/popmon/compare/v0.4.0...v0.4.1) (2021-06-23)
+
+
+### 🎉 Features
+
+* tabular traffic light/alerts overviews ([1d31265](https://github.com/ing-bank/popmon/commits/1d312653e0f2e788f4631839d201460f7e4ff562))
+
+
+### 👷‍♂️ Internal improvements
+
+* add utils to prevent code repetition ([d4a28cb](https://github.com/ing-bank/popmon/commits/d4a28cbfe6e3dcc8cf5ed1d92b1d679eb06aaab7))
+* convert dict constructor to dict literal ([331d8b1](https://github.com/ing-bank/popmon/commits/331d8b17620d90fef24360232fcdf2ab84e40b92)),  ([b3d4d88](https://github.com/ing-bank/popmon/commits/b3d4d889cc4a38a4f2a0d29d87c5ec4ba417cfbf)), ([9e5188c](https://github.com/ing-bank/popmon/commits/9e5188c80dd709885311a2e041a06c86c6caa897)), ([c9518ab](https://github.com/ing-bank/popmon/commits/c9518abc1e52789193d928e336eddb38d2b5881e))
+* directly pass arguments ([bf33807](https://github.com/ing-bank/popmon/commits/bf338075f4dee3c8e00e8997497e32063482b8f3))
+
+
+### 🐛 Bug fixes
+
+* copyright in license ([b16fd99](https://github.com/ing-bank/popmon/commits/b16fd993a0c4bed2ee50991ac2863e2f196c65ae)), ([60a22dd](https://github.com/ing-bank/popmon/commits/60a22dd36fa0ec9c2a6a6dc97eccb72509841f16))
+* extend TL config ([eba10ed](https://github.com/ing-bank/popmon/commits/eba10edb5d784c6a7644cef999d42bad22a8d7a2))
+* mismatch between stats displayed in the traffic lights section and comparisons due to pattern matching bug. ([5ffb0a9](https://github.com/ing-bank/popmon/commits/5ffb0a97e032f5500f65d01a88a93f2b60e99471))
+
+
+### 📖 Documentation
+
+* add explanation for hiding green TLs by default ([81095af](https://github.com/ing-bank/popmon/commits/81095af73b5b0157c4d6d373142a49747deda22b))
+* codemotion article ([1e20f30](https://github.com/ing-bank/popmon/commits/1e20f304567afd1edac987521c8026e2146f5d17))
+* include changelog in docs ([a2237d0](https://github.com/ing-bank/popmon/commits/a2237d088439ecd563fc68ad82696855bf8f6ec4))
+
+
+### ⬆️ Dependencies
+
+* pre-commit updates ([6f55155](https://github.com/ing-bank/popmon/commits/6f55155a920de83e33c6129ddd85ecaf682eff47)), ([8fc5518](https://github.com/ing-bank/popmon/commits/8fc5518c5963f6a9dfdcf5e7f8025faa69b0f983)), ([e02adf1](https://github.com/ing-bank/popmon/commits/e02adf1dab9e30815fec4150eedbb01e15a81509))
+* **histogrammar:** popmon working with hgr v1.0.23 ([#101](https://github.com/ing-bank/popmon/issues/101)) ([d4a986e](https://github.com/ing-bank/popmon/commits/d4a986ed0add983721d2a60fbefc385fe2ed7ed3))
+* Pin ing-matplotlib-theme dependency to master (closes #131) ([1e20f30](https://github.com/ing-bank/popmon/commit/f1ed364045d4286ab3e144034fd8cf2c1e3aef89))

CHANGES.rst

@@ -2,20 +2,22 @@
 Release notes
 =============
 
-Version 0.4.0, (16-04-2021)
----------------------------
+The release notes for more recent versions have move to `CHANGELOG.md <CHANGELOG.md>`_
+
+Version 0.4.0 (16-04-2021)
+--------------------------
 Documentation:
 
-* **docs**: include BDTWS presentation
-* **docs**: clarify that ``time_axis`` should be date or numeric
-* **docs**: initialize spark with both histogrammar jar files
+* Include BDTWS presentation
+* Clarify that ``time_axis`` should be date or numeric
+* Initialize spark with both histogrammar jar files
 
 Build system
 
-* **build**: Migrate to version 1.0.25 of ``histogrammar``.
-* **build**: update ``pyupgrade`` to v2.12.0
-* **build**: update ``isort`` to 5.8.0
-* **build**: update ``flake8`` to 3.9.0
+* Migrate to version 1.0.25 of ``histogrammar``.
+* Update ``pyupgrade`` to v2.12.0
+* Update ``isort`` to 5.8.0
+* Update ``flake8`` to 3.9.0
 
 Version 0.3.14, Feb 2021
 ------------------------

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2020 ING Wholesale Banking Advanced Analytics
+Copyright 2021 ING Wholesale Banking Advanced Analytics
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
 documentation files (the "Software"), to deal in the Software without restriction, including without limitation

Makefile

@@ -3,3 +3,6 @@ lint:
 
 install:
 	pip install -e .
+
+changelog:
+	npm run release

README.rst

@@ -172,6 +172,8 @@ Articles
 +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------+----------------+
 | Title                                                                                                                                                                                             | Date             | Author         |
 +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------+----------------+
+| `Population Shift Analysis: Monitoring Data Quality with Popmon <https://www.codemotion.com/magazine/dev-hub/big-data-analyst/popmon-data-quality-monitoring/>`_                                  | May 21, 2021     | Vito Gentile   |
++---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------+----------------+
 | `Popmon Open Source Package — Population Shift Monitoring Made Easy <https://medium.com/wbaa/population-monitoring-open-source-1ce3139d8c3a>`_                                                    | May 20, 2020     | Nicole Mpozika |
 +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------+----------------+
 

docs/source/changelog.rst

@@ -0,0 +1 @@
+.. include:: ../../CHANGES.rst

docs/source/index.rst

@@ -18,6 +18,7 @@ Contents
    configuration
    tree
    developing
+   changelog
 
 API
 ---

make.bat

@@ -12,5 +12,10 @@ IF "%1%" == "install" (
 	GOTO end
 )
 
+if "%1%" == "changelog" (
+    npm run release
+    GOTO end
+)
+
 ECHO "No command matched"
 :end

package.json

@@ -0,0 +1,12 @@
+{
+  "scripts": {
+    "release": "standard-version"
+  },
+  "standard-version": {
+    "skip": {
+      "bump": true,
+      "commit": true,
+      "tag": true
+    }
+  }
+}

popmon/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

popmon/alerting/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

popmon/alerting/alerts_summary.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

popmon/alerting/compute_tl_bounds.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in
@@ -42,11 +42,9 @@ def traffic_light_summary(row, cols=None, prefix=""):
     :param list cols: list of cols to calculate traffic light summary of (optional)
     :param str prefix: prefix of traffic light columns, in case cols is empty. default is ``"tl_"``
     """
-    x = pd.Series()
-    x["worst"] = np.nan
-    x["n_red"] = np.nan
-    x["n_yellow"] = np.nan
-    x["n_green"] = np.nan
+    x = pd.Series(
+        {"worst": np.nan, "n_red": np.nan, "n_yellow": np.nan, "n_green": np.nan}
+    )
 
     if cols is None or len(cols) == 0:
         # if no columns are given, find traffic light columns for which summary is made.
@@ -276,7 +274,7 @@ def pull_bounds(
     ).sum() == 0, "Traffic lights not sorted!"
 
     if cols is None or len(cols) == 0:
-        # if no columns are given, find colums for which pulls can be calculated.
+        # if no columns are given, find columns for which pulls can be calculated.
         # e.g. to calculate x_pull, need to have [x, x_mean, x_std] present. If so, put x in cols.
         cols = []
         for m in row.index.to_list():

popmon/analysis/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

popmon/analysis/apply_func.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in
@@ -58,7 +58,7 @@ def __init__(
         :param list apply_funcs: functions to apply (list of dicts):
 
           - 'func': function to apply
-          - 'suffix' (string, optional): suffix added to each metric. default is fuction name.
+          - 'suffix' (string, optional): suffix added to each metric. default is function name.
           - 'prefix' (string, optional): prefix added to each metric.
           - 'features' (list, optional): features the function is applied to. Overwrites features above
           - 'metrics' (list, optional): metrics the function is applied to. Overwrites metrics above
@@ -106,7 +106,7 @@ def add_apply_func(
         :param kwargs: (dict, optional) kwargs for 'func'
         """
         # check inputs
-        if not isinstance(func, collections.Callable):
+        if not callable(func):
             raise TypeError("functions in ApplyFunc must be callable objects")
         if suffix is not None and not isinstance(suffix, str):
             raise TypeError("prefix, and suffix in ApplyFunc must be strings or None.")

popmon/analysis/comparison/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in