Merge pull request #190 from ing-bank/develop

Release
ing-bank · Apr 29, 2022 · 2759f38 · 2759f38
2 parents 78073fe + aff68b7
commit 2759f38
Show file tree

Hide file tree

Showing 57 changed files with 110 additions and 102 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -11,7 +11,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python: [3.6, 3.7, 3.8]
+        python: [3.7, 3.8, 3.9]
     runs-on: ${{ matrix.os }}
 
     steps:
@@ -36,7 +36,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python: [3.6]
+        python: [3.7]
     runs-on: ${{ matrix.os }}
 
     steps:
@@ -77,10 +77,10 @@ jobs:
     - test_spark
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Python 3.6
+      - name: Set up Python 3.7
         uses: actions/setup-python@v1
         with:
-          python-version: 3.6
+          python-version: 3.7
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

diff --git a/.github/workflows/license.yml b/.github/workflows/license.yml
@@ -23,7 +23,7 @@ jobs:
     - name: Install addlicense
       run: |
         export PATH=${PATH}:`go env GOPATH`/bin
-        go get -v -u github.com/google/addlicense
+        go install github.com/google/addlicense@latest
     - name: Check license
       run: |
         export PATH=${PATH}:`go env GOPATH`/bin

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,6 +1,6 @@
 repos:
 -   repo: https://github.com/psf/black
-    rev: 21.12b0
+    rev: 22.3.0
     hooks:
     - id: black
 -   repo: https://github.com/pycqa/isort
@@ -18,11 +18,11 @@ repos:
             - tryceratops
         args: [ "--select=E9,F63,F7,F82,C4,F401,TR004,TC200,TC201,TC202"]
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.29.1
+    rev: v2.32.0
     hooks:
     -   id: pyupgrade
         args: ['--py36-plus','--exit-zero-even-if-changed']
 -   repo: https://github.com/asottile/blacken-docs
-    rev: v1.12.0
+    rev: v1.12.1
     hooks:
     -   id: blacken-docs
diff --git a/LICENSE b/LICENSE
@@ -1,4 +1,4 @@
-Copyright 2021 ING Wholesale Banking Advanced Analytics
+Copyright 2022 ING Wholesale Banking Advanced Analytics
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
 documentation files (the "Software"), to deal in the Software without restriction, including without limitation

diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst
@@ -243,24 +243,34 @@ Now that spark is installed, restart the runtime.
 
 .. code-block:: python
 
-    import os
+  import os
 
-    os.environ["JAVA_HOME"] = "/usr/lib/jvm/java-8-openjdk-amd64"
-    os.environ["SPARK_HOME"] = "/content/spark-2.4.7-bin-hadoop2.7"
+  os.environ["JAVA_HOME"] = "/usr/lib/jvm/java-8-openjdk-amd64"
+  os.environ["SPARK_HOME"] = "/content/spark-2.4.7-bin-hadoop2.7"
 
-    import findspark
+  import findspark
 
-    findspark.init()
+  findspark.init()
 
-    from pyspark.sql import SparkSession
+  from pyspark.sql import SparkSession
+
+  spark = (
+      SparkSession.builder.master("local[*]")
+      .config(
+          "spark.jars",
+          "/content/jars/histogrammar_2.12-1.0.20.jar,/content/jars/histogrammar-sparksql_2.12-1.0.20.jar",
+      )
+      .config("spark.sql.execution.arrow.enabled", "false")
+      .config("spark.sql.session.timeZone", "GMT")
+      .getOrCreate()
+  )
+Troubleshooting Spark
+~~~~~~~~~~~~~~~~~~~~~
+
+The following section documents error that you may run into using spark, and how they can be resolved.
+
+    TypeError: 'JavaPackage' object is not callable 
+
+This error occurs when pyspark cannot find the required jars. Ensure that the location of "spark.jars" or "spark.jars.packages" is correct (see examples above). Stop the spark session and restart it with the exact location. You can use the ones hosted on github using the "spark.jars.pacakges" example or place the jars locally and use "spark.jars", depending on whichever is easiest in your setup.
 
-    spark = (
-        SparkSession.builder.master("local[*]")
-        .config(
-            "spark.jars",
-            "/content/jars/histogrammar_2.12-1.0.20.jar,/content/jars/histogrammar-sparksql_2.12-1.0.20.jar",
-        )
-        .config("spark.sql.execution.arrow.enabled", "false")
-        .config("spark.sql.session.timeZone", "GMT")
-        .getOrCreate()
-    )
+If you are running in a jupyter notebook, then the kernel needs to be restarted.
diff --git a/popmon/__init__.py b/popmon/__init__.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/alerting/__init__.py b/popmon/alerting/__init__.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/alerting/alerts_summary.py b/popmon/alerting/alerts_summary.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/alerting/compute_tl_bounds.py b/popmon/alerting/compute_tl_bounds.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/analysis/__init__.py b/popmon/analysis/__init__.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/analysis/apply_func.py b/popmon/analysis/apply_func.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/analysis/comparison/__init__.py b/popmon/analysis/comparison/__init__.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/analysis/comparison/comparisons.py b/popmon/analysis/comparison/comparisons.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/analysis/comparison/hist_comparer.py b/popmon/analysis/comparison/hist_comparer.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/analysis/functions.py b/popmon/analysis/functions.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/analysis/hist_numpy.py b/popmon/analysis/hist_numpy.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/analysis/merge_statistics.py b/popmon/analysis/merge_statistics.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/analysis/profiling/__init__.py b/popmon/analysis/profiling/__init__.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/analysis/profiling/hist_profiler.py b/popmon/analysis/profiling/hist_profiler.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/analysis/profiling/pull_calculator.py b/popmon/analysis/profiling/pull_calculator.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/base/__init__.py b/popmon/base/__init__.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/base/module.py b/popmon/base/module.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/base/pipeline.py b/popmon/base/pipeline.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/config.py b/popmon/config.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/decorators/__init__.py b/popmon/decorators/__init__.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/decorators/pandas.py b/popmon/decorators/pandas.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/decorators/spark.py b/popmon/decorators/spark.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/hist/__init__.py b/popmon/hist/__init__.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/hist/filling/__init__.py b/popmon/hist/filling/__init__.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/hist/hist_splitter.py b/popmon/hist/hist_splitter.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/hist/hist_utils.py b/popmon/hist/hist_utils.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/io/__init__.py b/popmon/io/__init__.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/io/file_reader.py b/popmon/io/file_reader.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/io/file_writer.py b/popmon/io/file_writer.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/io/json_reader.py b/popmon/io/json_reader.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/pipeline/__init__.py b/popmon/pipeline/__init__.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/pipeline/amazing_pipeline.py b/popmon/pipeline/amazing_pipeline.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/pipeline/metrics.py b/popmon/pipeline/metrics.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/pipeline/metrics_pipelines.py b/popmon/pipeline/metrics_pipelines.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/pipeline/report.py b/popmon/pipeline/report.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/pipeline/report_pipelines.py b/popmon/pipeline/report_pipelines.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/resources.py b/popmon/resources.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/stats/__init__.py b/popmon/stats/__init__.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/stats/numpy.py b/popmon/stats/numpy.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/stitching/__init__.py b/popmon/stitching/__init__.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/stitching/hist_stitcher.py b/popmon/stitching/hist_stitcher.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/utils.py b/popmon/utils.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/visualization/__init__.py b/popmon/visualization/__init__.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/visualization/alert_section_generator.py b/popmon/visualization/alert_section_generator.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/visualization/backend.py b/popmon/visualization/backend.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

diff --git a/popmon/visualization/histogram_section.py b/popmon/visualization/histogram_section.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in