From a2237d088439ecd563fc68ad82696855bf8f6ec4 Mon Sep 17 00:00:00 2001 From: sbrugman Date: Fri, 16 Apr 2021 17:25:18 +0200 Subject: [PATCH 01/30] docs: include changelog in docs --- docs/source/changelog.rst | 1 + docs/source/index.rst | 1 + 2 files changed, 2 insertions(+) create mode 100644 docs/source/changelog.rst diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst new file mode 100644 index 00000000..d76c92b6 --- /dev/null +++ b/docs/source/changelog.rst @@ -0,0 +1 @@ +.. include:: ../../CHANGES.rst diff --git a/docs/source/index.rst b/docs/source/index.rst index 0c151a9c..4391f48c 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -18,6 +18,7 @@ Contents configuration tree developing + changelog API --- From 70551abc2c1457a72e8b41fc8a7e4cb46d019665 Mon Sep 17 00:00:00 2001 From: sbrugman Date: Wed, 14 Apr 2021 20:51:50 +0200 Subject: [PATCH 02/30] feat: turn off alert metric plots by default in config they do not provide additional information to the overview --- popmon/visualization/alert_section_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/popmon/visualization/alert_section_generator.py b/popmon/visualization/alert_section_generator.py index 47405dc9..3f5d1733 100644 --- a/popmon/visualization/alert_section_generator.py +++ b/popmon/visualization/alert_section_generator.py @@ -94,7 +94,7 @@ def __init__( self.description = description self.show_stats = show_stats self.plot_overview = True - self.plot_metrics = True + self.plot_metrics = False def transform(self, datastore): data_obj = self.get_datastore_object(datastore, self.read_key, dtype=dict) From 9082f2c3cc723c2bdb6f9b3437ef7c82ae5254c4 Mon Sep 17 00:00:00 2001 From: sbrugman Date: Thu, 15 Apr 2021 01:18:53 +0200 Subject: [PATCH 03/30] fix: copyright --- popmon/visualization/templates/footer.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/popmon/visualization/templates/footer.html b/popmon/visualization/templates/footer.html index 559fceb6..41ee0cea 100644 --- a/popmon/visualization/templates/footer.html +++ b/popmon/visualization/templates/footer.html @@ -1,7 +1,7 @@
-

Copyright © ING WBAA 2020

+

Copyright © ING WBAA 2021

From 73101e6d00b25b58f83a556c60cd7a019fe50f7c Mon Sep 17 00:00:00 2001 From: sbrugman Date: Thu, 15 Apr 2021 01:19:01 +0200 Subject: [PATCH 04/30] fix: typos --- popmon/alerting/compute_tl_bounds.py | 2 +- popmon/resources.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/popmon/alerting/compute_tl_bounds.py b/popmon/alerting/compute_tl_bounds.py index 9ec6d9e9..230afb77 100644 --- a/popmon/alerting/compute_tl_bounds.py +++ b/popmon/alerting/compute_tl_bounds.py @@ -276,7 +276,7 @@ def pull_bounds( ).sum() == 0, "Traffic lights not sorted!" if cols is None or len(cols) == 0: - # if no columns are given, find colums for which pulls can be calculated. + # if no columns are given, find columns for which pulls can be calculated. # e.g. to calculate x_pull, need to have [x, x_mean, x_std] present. If so, put x in cols. cols = [] for m in row.index.to_list(): diff --git a/popmon/resources.py b/popmon/resources.py index 2b2bd453..bc96dc1a 100644 --- a/popmon/resources.py +++ b/popmon/resources.py @@ -101,7 +101,7 @@ def templates_env(filename=None, **kwargs): :param str filename: the name of the template to get retrieved. :param kwargs: residual keyword arguments which would be used for rendering - :returns: template if a filename is provided (rendered given that keyword arguemnts are provided) + :returns: template if a filename is provided (rendered given that keyword arguments are provided) otherwise: environment of the templates directory """ if filename: From 51d7a088cc7cd0aae2032fde7c2cc0a8b7faec76 Mon Sep 17 00:00:00 2001 From: sbrugman Date: Thu, 15 Apr 2021 01:32:39 +0200 Subject: [PATCH 05/30] fix: copyright in license --- LICENSE | 2 +- popmon/__init__.py | 2 +- popmon/alerting/__init__.py | 2 +- popmon/alerting/alerts_summary.py | 2 +- popmon/alerting/compute_tl_bounds.py | 2 +- popmon/analysis/__init__.py | 2 +- popmon/analysis/apply_func.py | 2 +- popmon/analysis/comparison/__init__.py | 2 +- popmon/analysis/comparison/hist_comparer.py | 2 +- popmon/analysis/functions.py | 2 +- popmon/analysis/hist_numpy.py | 2 +- popmon/analysis/merge_statistics.py | 2 +- popmon/analysis/profiling/__init__.py | 2 +- popmon/analysis/profiling/hist_profiler.py | 2 +- popmon/analysis/profiling/pull_calculator.py | 2 +- popmon/base/__init__.py | 2 +- popmon/base/module.py | 2 +- popmon/base/pipeline.py | 2 +- popmon/config.py | 2 +- popmon/decorators/__init__.py | 2 +- popmon/decorators/pandas.py | 2 +- popmon/decorators/spark.py | 2 +- popmon/hist/__init__.py | 2 +- popmon/hist/filling/__init__.py | 2 +- popmon/hist/hist_splitter.py | 2 +- popmon/hist/hist_utils.py | 2 +- popmon/io/__init__.py | 2 +- popmon/io/file_reader.py | 2 +- popmon/io/file_writer.py | 2 +- popmon/io/json_reader.py | 2 +- popmon/pipeline/__init__.py | 2 +- popmon/pipeline/amazing_pipeline.py | 2 +- popmon/pipeline/metrics.py | 2 +- popmon/pipeline/metrics_pipelines.py | 2 +- popmon/pipeline/report.py | 2 +- popmon/pipeline/report_pipelines.py | 2 +- popmon/resources.py | 2 +- popmon/stats/__init__.py | 2 +- popmon/stats/numpy.py | 2 +- popmon/stitching/__init__.py | 2 +- popmon/stitching/hist_stitcher.py | 2 +- popmon/visualization/__init__.py | 2 +- popmon/visualization/alert_section_generator.py | 2 +- popmon/visualization/backend.py | 2 +- popmon/visualization/histogram_section.py | 2 +- popmon/visualization/report_generator.py | 2 +- popmon/visualization/section_generator.py | 2 +- popmon/visualization/traffic_light_section_generator.py | 2 +- popmon/visualization/utils.py | 2 +- 49 files changed, 49 insertions(+), 49 deletions(-) diff --git a/LICENSE b/LICENSE index 5583c295..e8210ddd 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright 2020 ING Wholesale Banking Advanced Analytics +Copyright 2021 ING Wholesale Banking Advanced Analytics Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation diff --git a/popmon/__init__.py b/popmon/__init__.py index 80bbedbf..f8ee496d 100644 --- a/popmon/__init__.py +++ b/popmon/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/alerting/__init__.py b/popmon/alerting/__init__.py index 730ffa38..c40e9b6b 100644 --- a/popmon/alerting/__init__.py +++ b/popmon/alerting/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/alerting/alerts_summary.py b/popmon/alerting/alerts_summary.py index 6ecbbfbb..88a35343 100644 --- a/popmon/alerting/alerts_summary.py +++ b/popmon/alerting/alerts_summary.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/alerting/compute_tl_bounds.py b/popmon/alerting/compute_tl_bounds.py index 230afb77..fa83825e 100644 --- a/popmon/alerting/compute_tl_bounds.py +++ b/popmon/alerting/compute_tl_bounds.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/analysis/__init__.py b/popmon/analysis/__init__.py index d9af6118..95645e1f 100644 --- a/popmon/analysis/__init__.py +++ b/popmon/analysis/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/analysis/apply_func.py b/popmon/analysis/apply_func.py index 339e5032..af70cbcc 100644 --- a/popmon/analysis/apply_func.py +++ b/popmon/analysis/apply_func.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/analysis/comparison/__init__.py b/popmon/analysis/comparison/__init__.py index 3802dacb..582ed99c 100644 --- a/popmon/analysis/comparison/__init__.py +++ b/popmon/analysis/comparison/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/analysis/comparison/hist_comparer.py b/popmon/analysis/comparison/hist_comparer.py index e45731f4..310baeab 100644 --- a/popmon/analysis/comparison/hist_comparer.py +++ b/popmon/analysis/comparison/hist_comparer.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/analysis/functions.py b/popmon/analysis/functions.py index 54ff8958..75a2938b 100644 --- a/popmon/analysis/functions.py +++ b/popmon/analysis/functions.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/analysis/hist_numpy.py b/popmon/analysis/hist_numpy.py index 8ff7fcad..937081d8 100644 --- a/popmon/analysis/hist_numpy.py +++ b/popmon/analysis/hist_numpy.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/analysis/merge_statistics.py b/popmon/analysis/merge_statistics.py index 4da31911..ccc435b9 100644 --- a/popmon/analysis/merge_statistics.py +++ b/popmon/analysis/merge_statistics.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/analysis/profiling/__init__.py b/popmon/analysis/profiling/__init__.py index 07a5fc9a..19c883ed 100644 --- a/popmon/analysis/profiling/__init__.py +++ b/popmon/analysis/profiling/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/analysis/profiling/hist_profiler.py b/popmon/analysis/profiling/hist_profiler.py index 93580e30..5d4bb02a 100644 --- a/popmon/analysis/profiling/hist_profiler.py +++ b/popmon/analysis/profiling/hist_profiler.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/analysis/profiling/pull_calculator.py b/popmon/analysis/profiling/pull_calculator.py index 166cb85b..17936872 100644 --- a/popmon/analysis/profiling/pull_calculator.py +++ b/popmon/analysis/profiling/pull_calculator.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/base/__init__.py b/popmon/base/__init__.py index 98f987be..9093605f 100644 --- a/popmon/base/__init__.py +++ b/popmon/base/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/base/module.py b/popmon/base/module.py index 9fe479d2..eed138da 100644 --- a/popmon/base/module.py +++ b/popmon/base/module.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/base/pipeline.py b/popmon/base/pipeline.py index f54749b4..31a83afe 100644 --- a/popmon/base/pipeline.py +++ b/popmon/base/pipeline.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/config.py b/popmon/config.py index ccccbdd3..4e8bb6bc 100644 --- a/popmon/config.py +++ b/popmon/config.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/decorators/__init__.py b/popmon/decorators/__init__.py index 6f8b8aab..90689539 100644 --- a/popmon/decorators/__init__.py +++ b/popmon/decorators/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/decorators/pandas.py b/popmon/decorators/pandas.py index dc88ed30..63b2fd98 100644 --- a/popmon/decorators/pandas.py +++ b/popmon/decorators/pandas.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/decorators/spark.py b/popmon/decorators/spark.py index 52a9cd8c..0af39333 100644 --- a/popmon/decorators/spark.py +++ b/popmon/decorators/spark.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/hist/__init__.py b/popmon/hist/__init__.py index c6e439f5..63e66af9 100644 --- a/popmon/hist/__init__.py +++ b/popmon/hist/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/hist/filling/__init__.py b/popmon/hist/filling/__init__.py index 333df22b..8a5b5985 100644 --- a/popmon/hist/filling/__init__.py +++ b/popmon/hist/filling/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/hist/hist_splitter.py b/popmon/hist/hist_splitter.py index bede5dce..4d11260e 100644 --- a/popmon/hist/hist_splitter.py +++ b/popmon/hist/hist_splitter.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/hist/hist_utils.py b/popmon/hist/hist_utils.py index 1c70a192..ae8f81b1 100644 --- a/popmon/hist/hist_utils.py +++ b/popmon/hist/hist_utils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/io/__init__.py b/popmon/io/__init__.py index 4988c618..b09bf248 100644 --- a/popmon/io/__init__.py +++ b/popmon/io/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/io/file_reader.py b/popmon/io/file_reader.py index 2738a84f..06c5e0f6 100644 --- a/popmon/io/file_reader.py +++ b/popmon/io/file_reader.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/io/file_writer.py b/popmon/io/file_writer.py index a5f1325a..2408b032 100644 --- a/popmon/io/file_writer.py +++ b/popmon/io/file_writer.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/io/json_reader.py b/popmon/io/json_reader.py index f169be8d..6fe4f7f2 100644 --- a/popmon/io/json_reader.py +++ b/popmon/io/json_reader.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/pipeline/__init__.py b/popmon/pipeline/__init__.py index b3174538..dbcd2fe7 100644 --- a/popmon/pipeline/__init__.py +++ b/popmon/pipeline/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/pipeline/amazing_pipeline.py b/popmon/pipeline/amazing_pipeline.py index b18d1482..a5dbb0ac 100644 --- a/popmon/pipeline/amazing_pipeline.py +++ b/popmon/pipeline/amazing_pipeline.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/pipeline/metrics.py b/popmon/pipeline/metrics.py index 57d9c5a1..821ff768 100644 --- a/popmon/pipeline/metrics.py +++ b/popmon/pipeline/metrics.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/pipeline/metrics_pipelines.py b/popmon/pipeline/metrics_pipelines.py index 336febff..1215cdf6 100644 --- a/popmon/pipeline/metrics_pipelines.py +++ b/popmon/pipeline/metrics_pipelines.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/pipeline/report.py b/popmon/pipeline/report.py index 203d2e58..158ad13b 100644 --- a/popmon/pipeline/report.py +++ b/popmon/pipeline/report.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/pipeline/report_pipelines.py b/popmon/pipeline/report_pipelines.py index 93a5498c..96df57d5 100644 --- a/popmon/pipeline/report_pipelines.py +++ b/popmon/pipeline/report_pipelines.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/resources.py b/popmon/resources.py index bc96dc1a..3bcb38ca 100644 --- a/popmon/resources.py +++ b/popmon/resources.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/stats/__init__.py b/popmon/stats/__init__.py index 61992a0f..c603bddf 100644 --- a/popmon/stats/__init__.py +++ b/popmon/stats/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/stats/numpy.py b/popmon/stats/numpy.py index d92dd584..231a2fad 100644 --- a/popmon/stats/numpy.py +++ b/popmon/stats/numpy.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/stitching/__init__.py b/popmon/stitching/__init__.py index 75e6b69a..4affaed9 100644 --- a/popmon/stitching/__init__.py +++ b/popmon/stitching/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/stitching/hist_stitcher.py b/popmon/stitching/hist_stitcher.py index fcdd55d1..54513d58 100644 --- a/popmon/stitching/hist_stitcher.py +++ b/popmon/stitching/hist_stitcher.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/visualization/__init__.py b/popmon/visualization/__init__.py index 2026687d..da7eecad 100644 --- a/popmon/visualization/__init__.py +++ b/popmon/visualization/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/visualization/alert_section_generator.py b/popmon/visualization/alert_section_generator.py index 3f5d1733..3a093aef 100644 --- a/popmon/visualization/alert_section_generator.py +++ b/popmon/visualization/alert_section_generator.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/visualization/backend.py b/popmon/visualization/backend.py index 3ea12e55..f209da95 100644 --- a/popmon/visualization/backend.py +++ b/popmon/visualization/backend.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/visualization/histogram_section.py b/popmon/visualization/histogram_section.py index 7a93adc5..4b8d2525 100644 --- a/popmon/visualization/histogram_section.py +++ b/popmon/visualization/histogram_section.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/visualization/report_generator.py b/popmon/visualization/report_generator.py index 232a2129..cc5ba4da 100644 --- a/popmon/visualization/report_generator.py +++ b/popmon/visualization/report_generator.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/visualization/section_generator.py b/popmon/visualization/section_generator.py index 9f81ab56..defd3a09 100644 --- a/popmon/visualization/section_generator.py +++ b/popmon/visualization/section_generator.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/visualization/traffic_light_section_generator.py b/popmon/visualization/traffic_light_section_generator.py index 22cd6e1f..21fdcf0f 100644 --- a/popmon/visualization/traffic_light_section_generator.py +++ b/popmon/visualization/traffic_light_section_generator.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/popmon/visualization/utils.py b/popmon/visualization/utils.py index 6071c7e5..30cdb89e 100644 --- a/popmon/visualization/utils.py +++ b/popmon/visualization/utils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020 ING Wholesale Banking Advanced Analytics +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in From c8cc8b0cfee1b11e11827d9355d41e14a541b5e1 Mon Sep 17 00:00:00 2001 From: sbrugman Date: Wed, 14 Apr 2021 23:48:11 +0200 Subject: [PATCH 06/30] fix: consistency between tl and comparisons mismatch between stats displayed in the traffic lights section and comparisons due to pattern matching bug. --- popmon/config.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/popmon/config.py b/popmon/config.py index 4e8bb6bc..4e585dce 100644 --- a/popmon/config.py +++ b/popmon/config.py @@ -21,7 +21,7 @@ from fnmatch import fnmatch profiles = { - "count": "Number of entries", + "count": "Number of entries (non-NaN and NaN)", "distinct": "Number of distinct entries", "filled": "Number of non-missing entries (non-NaN)", "nan": "Number of missing entries (NaN)", @@ -100,14 +100,13 @@ "*ks", "*max_prob_diff", "*zscore", - "*pull", "n_*", "worst", ], } -def get_stat_description(name): +def get_stat_description(name: str): """Gets the description of a statistic. :param str name: the name of the statistic. From a51fabfdd70f05d99f428b7d7c5bc8afec13c63c Mon Sep 17 00:00:00 2001 From: sbrugman Date: Thu, 15 Apr 2021 01:24:02 +0200 Subject: [PATCH 07/30] refactor: replace `dict()` with `{}` --- popmon/analysis/comparison/hist_comparer.py | 23 ++++--- popmon/analysis/merge_statistics.py | 2 +- popmon/analysis/profiling/hist_profiler.py | 24 +++---- .../notebooks/popmon_tutorial_advanced.ipynb | 4 +- popmon/pipeline/metrics.py | 3 +- popmon/pipeline/metrics_pipelines.py | 64 +++++++++---------- popmon/pipeline/report.py | 13 ++-- popmon/pipeline/report_pipelines.py | 20 +++--- popmon/resources.py | 2 +- popmon/stitching/hist_stitcher.py | 4 +- .../visualization/alert_section_generator.py | 4 +- popmon/visualization/histogram_section.py | 4 +- popmon/visualization/report_generator.py | 7 +- popmon/visualization/section_generator.py | 4 +- .../traffic_light_section_generator.py | 6 +- 15 files changed, 99 insertions(+), 85 deletions(-) diff --git a/popmon/analysis/comparison/hist_comparer.py b/popmon/analysis/comparison/hist_comparer.py index 310baeab..df617f7b 100644 --- a/popmon/analysis/comparison/hist_comparer.py +++ b/popmon/analysis/comparison/hist_comparer.py @@ -177,14 +177,14 @@ def __init__( apply_to_key=assign_to_key, assign_to_key=store_key, apply_funcs=[ - dict( - func=hist_compare, - hist_name1=hist_col, - hist_name2=hist_col + "_" + suffix, - prefix=suffix, - axis=1, - max_res_bound=max_res_bound, - ) + { + "func": hist_compare, + "hist_name1": hist_col, + "hist_name2": hist_col + "_" + suffix, + "prefix": suffix, + "axis": 1, + "max_res_bound": max_res_bound, + } ], ) self.modules = [hist_collector, hist_comparer] @@ -382,7 +382,12 @@ def __init__( apply_to_key=assign_to_key, assign_to_key=store_key, apply_funcs=[ - dict(func=relative_chi_squared, hist_name=hist_col, suffix="", axis=1) + { + "func": relative_chi_squared, + "hist_name": hist_col, + "suffix": "", + "axis": 1, + } ], ) diff --git a/popmon/analysis/merge_statistics.py b/popmon/analysis/merge_statistics.py index ccc435b9..188158b3 100644 --- a/popmon/analysis/merge_statistics.py +++ b/popmon/analysis/merge_statistics.py @@ -41,7 +41,7 @@ def transform(self, datastore): self.get_datastore_object(datastore, read_key, dtype=dict) for read_key in self.read_keys ] - merged_stats = dict() + merged_stats = {} for dict_ in dicts: for feature in dict_.keys(): # we add statistics dataframe to the final output for specific feature however diff --git a/popmon/analysis/profiling/hist_profiler.py b/popmon/analysis/profiling/hist_profiler.py index 5d4bb02a..0671f90e 100644 --- a/popmon/analysis/profiling/hist_profiler.py +++ b/popmon/analysis/profiling/hist_profiler.py @@ -106,28 +106,30 @@ def _profile_1d_histogram(self, name, hist): if len(bin_counts) == 0: self.logger.warning(f'Histogram "{name}" is empty; skipping.') - return dict() + return {} if is_ts: to_timestamp = np.vectorize(lambda x: pd.to_datetime(x).value) bin_labels = to_timestamp(bin_labels) - profile = dict() - profile["filled"] = bin_counts.sum() + profile = { + "filled": bin_counts.sum(), + "overflow": hist.overflow.entries if hasattr(hist, "overflow") else 0, + "underflow": (hist.underflow.entries if hasattr(hist, "underflow") else 0), + "distinct": len(np.unique(bin_labels)), + } + if hasattr(hist, "nanflow"): profile["nan"] = hist.nanflow.entries elif hasattr(hist, "bins") and "NaN" in hist.bins: profile["nan"] = hist.bins["NaN"].entries else: profile["nan"] = 0 - profile["overflow"] = hist.overflow.entries if hasattr(hist, "overflow") else 0 - profile["underflow"] = ( - hist.underflow.entries if hasattr(hist, "underflow") else 0 - ) profile["count"] = profile["filled"] + profile["nan"] - profile["distinct"] = len(np.unique(bin_labels)) - mpv = bin_labels[np.argmax(bin_counts)] # most probable value - profile["most_probable_value"] = mpv if not is_ts else pd.Timestamp(mpv) + most_probable_value = bin_labels[np.argmax(bin_counts)] + profile["most_probable_value"] = ( + most_probable_value if not is_ts else pd.Timestamp(most_probable_value) + ) if is_num and profile["filled"] > 0: for f_names, func in self.stats_functions.items(): @@ -225,7 +227,7 @@ def transform(self, datastore): f'Profiling histograms "{self.read_key}" as "{self.store_key}"' ) data = self.get_datastore_object(datastore, self.read_key, dtype=dict) - profiled = dict() + profiled = {} features = self.get_features(data.keys()) diff --git a/popmon/notebooks/popmon_tutorial_advanced.ipynb b/popmon/notebooks/popmon_tutorial_advanced.ipynb index e2fa7107..208d3a3b 100644 --- a/popmon/notebooks/popmon_tutorial_advanced.ipynb +++ b/popmon/notebooks/popmon_tutorial_advanced.ipynb @@ -389,7 +389,7 @@ " \"*_zscore\": [7, 4, -4, -7],\n", " \"[!p]*_unknown_labels\": [0.5, 0.5, 0, 0],\n", "}\n", - "datastore = dict()\n", + "datastore = {}\n", "datastore[\"hists\"] = df.pm_make_histograms(\n", " time_axis=\"DATE\", time_width=\"1w\", time_offset=\"2015-07-02\"\n", ")\n", @@ -425,7 +425,7 @@ "source": [ "from popmon.analysis.comparison.hist_comparer import ReferenceHistComparer\n", "\n", - "datastore = dict()\n", + "datastore = {}\n", "datastore[\"hists\"] = df.pm_make_histograms(\n", " time_axis=\"DATE\", time_width=\"1w\", time_offset=\"2015-07-02\"\n", ")\n", diff --git a/popmon/pipeline/metrics.py b/popmon/pipeline/metrics.py index 821ff768..aa79604b 100644 --- a/popmon/pipeline/metrics.py +++ b/popmon/pipeline/metrics.py @@ -146,8 +146,7 @@ def stability_metrics( } cfg.update(kwargs) - datastore = dict() - datastore["hists"] = hists + datastore = {"hists": hists} if reference_type == "external": datastore["ref_hists"] = reference diff --git a/popmon/pipeline/metrics_pipelines.py b/popmon/pipeline/metrics_pipelines.py index 1215cdf6..3de19b23 100644 --- a/popmon/pipeline/metrics_pipelines.py +++ b/popmon/pipeline/metrics_pipelines.py @@ -104,13 +104,13 @@ def metrics_self_reference( apply_to_key="profiles", assign_to_key="comparisons", apply_funcs=[ - dict( - func=rolling_lr_zscore, - suffix=f"_trend{window}_zscore", - entire=True, - window=window, - metrics=["mean", "phik", "fraction_true"], - ) + { + "func": rolling_lr_zscore, + "suffix": f"_trend{window}_zscore", + "entire": True, + "window": window, + "metrics": ["mean", "phik", "fraction_true"], + } ], msg="Computing significance of (rolling) trend in means of features", ), @@ -146,7 +146,7 @@ def metrics_self_reference( ), ApplyFunc( apply_to_key="traffic_lights", - apply_funcs=[dict(func=traffic_light_summary, axis=1, suffix="")], + apply_funcs=[{"func": traffic_light_summary, "axis": 1, "suffix": ""}], assign_to_key="alerts", msg="Generating traffic light alerts summary.", ), @@ -225,13 +225,13 @@ def metrics_external_reference( apply_to_key="profiles", assign_to_key="comparisons", apply_funcs=[ - dict( - func=rolling_lr_zscore, - suffix=f"_trend{window}_zscore", - entire=True, - window=window, - metrics=["mean", "phik", "fraction_true"], - ) + { + "func": rolling_lr_zscore, + "suffix": f"_trend{window}_zscore", + "entire": True, + "window": window, + "metrics": ["mean", "phik", "fraction_true"], + } ], msg="Computing significance of (rolling) trend in means of features", ), @@ -267,7 +267,7 @@ def metrics_external_reference( ), ApplyFunc( apply_to_key="traffic_lights", - apply_funcs=[dict(func=traffic_light_summary, axis=1, suffix="")], + apply_funcs=[{"func": traffic_light_summary, "axis": 1, "suffix": ""}], assign_to_key="alerts", msg="Generating traffic light alerts summary.", ), @@ -340,13 +340,13 @@ def metrics_rolling_reference( apply_to_key="profiles", assign_to_key="comparisons", apply_funcs=[ - dict( - func=rolling_lr_zscore, - suffix=f"_trend{window}_zscore", - entire=True, - window=window, - metrics=["mean", "phik", "fraction_true"], - ) + { + "func": rolling_lr_zscore, + "suffix": f"_trend{window}_zscore", + "entire": True, + "window": window, + "metrics": ["mean", "phik", "fraction_true"], + } ], msg="Computing significance of (rolling) trend in means of features", ), @@ -382,7 +382,7 @@ def metrics_rolling_reference( ), ApplyFunc( apply_to_key="traffic_lights", - apply_funcs=[dict(func=traffic_light_summary, axis=1, suffix="")], + apply_funcs=[{"func": traffic_light_summary, "axis": 1, "suffix": ""}], assign_to_key="alerts", msg="Generating traffic light alerts summary.", ), @@ -454,13 +454,13 @@ def metrics_expanding_reference( apply_to_key="profiles", assign_to_key="comparisons", apply_funcs=[ - dict( - func=rolling_lr_zscore, - suffix=f"_trend{window}_zscore", - entire=True, - window=window, - metrics=["mean", "phik", "fraction_true"], - ) + { + "func": rolling_lr_zscore, + "suffix": f"_trend{window}_zscore", + "entire": True, + "window": window, + "metrics": ["mean", "phik", "fraction_true"], + } ], msg="Computing significance of (rolling) trend in means of features", ), @@ -496,7 +496,7 @@ def metrics_expanding_reference( ), ApplyFunc( apply_to_key="traffic_lights", - apply_funcs=[dict(func=traffic_light_summary, axis=1, suffix="")], + apply_funcs=[{"func": traffic_light_summary, "axis": 1, "suffix": ""}], assign_to_key="alerts", msg="Generating traffic light alerts summary.", ), diff --git a/popmon/pipeline/report.py b/popmon/pipeline/report.py index 158ad13b..54906811 100644 --- a/popmon/pipeline/report.py +++ b/popmon/pipeline/report.py @@ -170,8 +170,7 @@ def stability_report( } cfg.update(kwargs) - datastore = dict() - datastore["hists"] = hists + datastore = {"hists": hists} if reference_type == "external": datastore["ref_hists"] = reference @@ -474,8 +473,14 @@ def to_notebook_iframe(self, width="100%", height="100%"): from IPython.core.display import HTML # get iframe's snippet code, insert report's HTML code and display it as HTML - args = dict(src=self.to_html(escape=True), width=width, height=height) - return HTML(templates_env(filename="notebook_iframe.html", **args)) + return HTML( + templates_env( + filename="notebook_iframe.html", + src=self.to_html(escape=True), + width=width, + height=height, + ) + ) def regenerate( self, diff --git a/popmon/pipeline/report_pipelines.py b/popmon/pipeline/report_pipelines.py index 96df57d5..2b66c8ac 100644 --- a/popmon/pipeline/report_pipelines.py +++ b/popmon/pipeline/report_pipelines.py @@ -312,16 +312,16 @@ def __init__( # default keyword arguments for each section def sg_kws(read_key): - return dict( - read_key=read_key, - store_key=sections_key, - skip_empty_plots=skip_empty_plots, - last_n=last_n, - skip_first_n=skip_first_n, - skip_last_n=skip_last_n, - show_stats=show_stats, - description=descs.get(read_key, ""), - ) + return { + "read_key": read_key, + "store_key": sections_key, + "skip_empty_plots": skip_empty_plots, + "last_n": last_n, + "skip_first_n": skip_first_n, + "skip_last_n": skip_last_n, + "show_stats": show_stats, + "description": descs.get(read_key, ""), + } self.modules = [ # --- o generate sections diff --git a/popmon/resources.py b/popmon/resources.py index 3bcb38ca..c15b996e 100644 --- a/popmon/resources.py +++ b/popmon/resources.py @@ -42,7 +42,7 @@ } # Resource types -_RESOURCES = dict(data=_DATA, notebook=_NOTEBOOK) +_RESOURCES = {"data": _DATA, "notebook": _NOTEBOOK} # Environment for visualization templates' directory _TEMPLATES_ENV = Environment( diff --git a/popmon/stitching/hist_stitcher.py b/popmon/stitching/hist_stitcher.py index 54513d58..2b095f6e 100644 --- a/popmon/stitching/hist_stitcher.py +++ b/popmon/stitching/hist_stitcher.py @@ -176,7 +176,7 @@ def stitch_histograms( f'Inserting basis histograms in axis "{time_axis}" at bin index 0.' ) time_bin_idx = [0] - hists_basis_new = dict() + hists_basis_new = {} for k, hist in hists_basis.items(): feature = f"{time_axis}:{k}" self.logger.debug(f'Now creating histogram "{feature}"') @@ -200,7 +200,7 @@ def stitch_histograms( ) return hists_basis - stitched = dict() + stitched = {} # 2. if delta(s) do not start with "time_axis:", assume that this is a set of batches without time_axis features_delta = self.get_features( diff --git a/popmon/visualization/alert_section_generator.py b/popmon/visualization/alert_section_generator.py index 3a093aef..d5de1d5b 100644 --- a/popmon/visualization/alert_section_generator.py +++ b/popmon/visualization/alert_section_generator.py @@ -182,7 +182,7 @@ def short_date(date): plots = [e for e in plots if len(e["plot"])] features_w_metrics.append( - dict(name=feature, plots=sorted(plots, key=lambda plot: plot["name"])) + {"name": feature, "plots": sorted(plots, key=lambda plot: plot["name"])} ) params = { @@ -241,4 +241,4 @@ def _plot_metric( skip_empty=skip_empty, ) - return dict(name=metric, description=get_stat_description(metric), plot=plot) + return {"name": metric, "description": get_stat_description(metric), "plot": plot} diff --git a/popmon/visualization/histogram_section.py b/popmon/visualization/histogram_section.py index 4b8d2525..d5d7b7aa 100644 --- a/popmon/visualization/histogram_section.py +++ b/popmon/visualization/histogram_section.py @@ -115,7 +115,7 @@ def short_date(date): # filter out potential empty plots plots = [e for e in plots if len(e["plot"])] features_w_metrics.append( - dict(name=feature, plots=sorted(plots, key=lambda plot: plot["name"])) + {"name": feature, "plots": sorted(plots, key=lambda plot: plot["name"])} ) params = { @@ -191,4 +191,4 @@ def _plot_histograms(feature, date, hc_list, hist_names): else: plot = "" - return dict(name=date, description=get_stat_description(date), plot=plot) + return {"name": date, "description": get_stat_description(date), "plot": plot} diff --git a/popmon/visualization/report_generator.py b/popmon/visualization/report_generator.py index cc5ba4da..ae17bdf6 100644 --- a/popmon/visualization/report_generator.py +++ b/popmon/visualization/report_generator.py @@ -51,8 +51,11 @@ def transform(self, datastore): ) # get HTML template for the final report, insert placeholder data and compress the code - args = dict(sections=sections_html) datastore[self.store_key] = htmlmin.minify( - templates_env(filename="core.html", generator=f"{name} {version}", **args) + templates_env( + filename="core.html", + generator=f"{name} {version}", + sections=sections_html, + ) ) return datastore diff --git a/popmon/visualization/section_generator.py b/popmon/visualization/section_generator.py index defd3a09..55bf2603 100644 --- a/popmon/visualization/section_generator.py +++ b/popmon/visualization/section_generator.py @@ -162,7 +162,7 @@ def short_date(date): if self.skip_empty_plots: plots = [e for e in plots if len(e["plot"])] features_w_metrics.append( - dict(name=feature, plots=sorted(plots, key=lambda plot: plot["name"])) + {"name": feature, "plots": sorted(plots, key=lambda plot: plot["name"])} ) params = { @@ -221,4 +221,4 @@ def _plot_metric( skip_empty=skip_empty, ) - return dict(name=metric, description=get_stat_description(metric), plot=plot) + return {"name": metric, "description": get_stat_description(metric), "plot": plot} diff --git a/popmon/visualization/traffic_light_section_generator.py b/popmon/visualization/traffic_light_section_generator.py index 21fdcf0f..184b4161 100644 --- a/popmon/visualization/traffic_light_section_generator.py +++ b/popmon/visualization/traffic_light_section_generator.py @@ -182,7 +182,7 @@ def short_date(date): if self.skip_empty_plots: plots = [e for e in plots if len(e["plot"])] features_w_metrics.append( - dict(name=feature, plots=sorted(plots, key=lambda plot: plot["name"])) + {"name": feature, "plots": sorted(plots, key=lambda plot: plot["name"])} ) params = { @@ -211,7 +211,7 @@ def _plot_metric(metric, dates, values, last_n, skip_first_n, skip_last_n, skip_ data=np.array(values), labels=dates, skip_empty=skip_empty ) - return dict(name=metric, description=get_stat_description(metric), plot=plot) + return {"name": metric, "description": get_stat_description(metric), "plot": plot} def _plot_metrics( @@ -248,4 +248,4 @@ def _plot_metrics( else: plot = "" - return dict(name="Overview", description="", plot=plot, full_width=True) + return {"name": "Overview", "description": "", "plot": plot, "full_width": True} From 47411ffd6f5e1ab49a66e9f4e87c474c56aa07ca Mon Sep 17 00:00:00 2001 From: sbrugman Date: Thu, 15 Apr 2021 01:24:27 +0200 Subject: [PATCH 08/30] refactor: directly pass arguments --- popmon/analysis/comparison/hist_comparer.py | 41 +++++++++++++++------ 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/popmon/analysis/comparison/hist_comparer.py b/popmon/analysis/comparison/hist_comparer.py index df617f7b..abdbc6ef 100644 --- a/popmon/analysis/comparison/hist_comparer.py +++ b/popmon/analysis/comparison/hist_comparer.py @@ -214,7 +214,6 @@ def __init__( :param float max_res_bound: count number of normalized residuals with (absolute) value greater than X. Default is 7.0. """ - kws = {"window": window, "shift": shift, "hist_name": hist_col} super().__init__( rolling_hist, read_key, @@ -223,7 +222,9 @@ def __init__( hist_col, suffix, max_res_bound, - **kws, + window=window, + shift=shift, + hist_name=hist_col, ) self.read_key = read_key self.window = window @@ -255,7 +256,15 @@ def __init__( :param float max_res_bound: count number of normalized residuals with (absolute) value greater than X. Default is 7.0. """ - super().__init__(read_key, store_key, 1, 1, hist_col, suffix, max_res_bound) + super().__init__( + read_key, + store_key, + window=1, + shift=1, + hist_col=hist_col, + suffix=suffix, + max_res_bound=max_res_bound, + ) class ExpandingHistComparer(HistComparer): @@ -280,7 +289,6 @@ def __init__( :param float max_res_bound: count number of normalized residuals with (absolute) value greater than X. Default is 7.0. """ - kws = {"shift": shift, "hist_name": hist_col} super().__init__( expanding_hist, read_key, @@ -289,7 +297,8 @@ def __init__( hist_col, suffix, max_res_bound, - **kws, + shift=shift, + hist_name=hist_col, ) self.read_key = read_key @@ -322,7 +331,6 @@ def __init__( :param float max_res_bound: count number of normalized residuals with (absolute) value greater than X. Default is 7.0. """ - kws = {"metrics": [hist_col]} super().__init__( hist_sum, reference_key, @@ -331,7 +339,7 @@ def __init__( hist_col, suffix, max_res_bound, - **kws, + metrics=[hist_col], ) self.reference_key = reference_key self.assign_to_key = assign_to_key @@ -408,9 +416,15 @@ def __init__(self, read_key, store_key, window, shift=1, hist_col="histogram"): """ if window < 2: raise ValueError("Need window size of 2 or greater.") - kws = {"window": window, "shift": shift, "entire": True} super().__init__( - roll_norm_hist_mean_cov, read_key, store_key, read_key, hist_col, **kws + roll_norm_hist_mean_cov, + read_key, + store_key, + read_key, + hist_col, + window=window, + shift=shift, + entire=True, ) self.read_key = read_key self.window = window @@ -433,9 +447,14 @@ def __init__(self, read_key, store_key, shift=1, hist_col="histogram"): :param int shift: shift of rolling window. default is 1. :param str hist_col: column/key in input df/dict that contains the histogram. default is 'histogram' """ - kws = {"shift": shift, "entire": True} super().__init__( - expand_norm_hist_mean_cov, read_key, store_key, read_key, hist_col, **kws + expand_norm_hist_mean_cov, + read_key, + store_key, + read_key, + hist_col, + shift=shift, + entire=True, ) self.read_key = read_key From 829911019f995adea963e2ea4bd6113b5b24f3b1 Mon Sep 17 00:00:00 2001 From: sbrugman Date: Thu, 15 Apr 2021 01:24:43 +0200 Subject: [PATCH 09/30] refactor: series from dict --- popmon/alerting/compute_tl_bounds.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/popmon/alerting/compute_tl_bounds.py b/popmon/alerting/compute_tl_bounds.py index fa83825e..ef7fb5bb 100644 --- a/popmon/alerting/compute_tl_bounds.py +++ b/popmon/alerting/compute_tl_bounds.py @@ -42,11 +42,9 @@ def traffic_light_summary(row, cols=None, prefix=""): :param list cols: list of cols to calculate traffic light summary of (optional) :param str prefix: prefix of traffic light columns, in case cols is empty. default is ``"tl_"`` """ - x = pd.Series() - x["worst"] = np.nan - x["n_red"] = np.nan - x["n_yellow"] = np.nan - x["n_green"] = np.nan + x = pd.Series( + {"worst": np.nan, "n_red": np.nan, "n_yellow": np.nan, "n_green": np.nan} + ) if cols is None or len(cols) == 0: # if no columns are given, find traffic light columns for which summary is made. From 8066bc32bdc02354cb283cf878971b30009108d3 Mon Sep 17 00:00:00 2001 From: sbrugman Date: Thu, 15 Apr 2021 01:25:07 +0200 Subject: [PATCH 10/30] refactor: add utils to prevent code repetition --- popmon/utils.py | 39 +++++++++++++++++++ .../visualization/alert_section_generator.py | 20 ++-------- popmon/visualization/histogram_section.py | 4 +- popmon/visualization/section_generator.py | 20 +++------- .../traffic_light_section_generator.py | 20 ++-------- 5 files changed, 53 insertions(+), 50 deletions(-) create mode 100644 popmon/utils.py diff --git a/popmon/utils.py b/popmon/utils.py new file mode 100644 index 00000000..35bbbeab --- /dev/null +++ b/popmon/utils.py @@ -0,0 +1,39 @@ +# Copyright (c) 2021 ING Wholesale Banking Advanced Analytics +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +# the Software, and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import fnmatch +from textwrap import shorten +from typing import Iterable, Optional + + +def short_date(date: str): + return shorten(date, width=22, placeholder="") + + +def filter_metrics(metrics, ignore_stat_endswith, show_stats: Optional[Iterable]): + metrics = [ + m for m in metrics if not any([m.endswith(s) for s in ignore_stat_endswith]) + ] + if show_stats is not None: + metrics = [ + m + for m in metrics + if any(fnmatch.fnmatch(m, pattern) for pattern in show_stats) + ] + return metrics diff --git a/popmon/visualization/alert_section_generator.py b/popmon/visualization/alert_section_generator.py index d5de1d5b..c8111def 100644 --- a/popmon/visualization/alert_section_generator.py +++ b/popmon/visualization/alert_section_generator.py @@ -18,7 +18,6 @@ # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -import fnmatch import multiprocessing import numpy as np @@ -28,6 +27,7 @@ from ..base import Module from ..config import get_stat_description +from ..utils import filter_metrics, short_date from ..visualization.utils import _prune, plot_bars_b64 from .traffic_light_section_generator import _plot_metrics @@ -115,9 +115,6 @@ def transform(self, datastore): f'Generating section "{self.section_name}". skip empty plots: {self.skip_empty_plots}' ) - def short_date(date): - return date if len(date) <= 22 else date[:22] - for feature in tqdm(features, ncols=100): df = data_obj.get(feature, pd.DataFrame()) fdbounds = dynamic_bounds.get(feature, pd.DataFrame(index=df.index)) @@ -132,18 +129,9 @@ def short_date(date): ) dates = [short_date(str(date)) for date in df.index.tolist()] - # get base64 encoded plot for each metric; do parallel processing to speed up. - metrics = [ - m - for m in df.columns - if not any([m.endswith(s) for s in self.ignore_stat_endswith]) - ] - if self.show_stats is not None: - metrics = [ - m - for m in metrics - if any(fnmatch.fnmatch(m, pattern) for pattern in self.show_stats) - ] + metrics = filter_metrics( + df.columns, self.ignore_stat_endswith, self.show_stats + ) plots = [] if self.plot_overview: diff --git a/popmon/visualization/histogram_section.py b/popmon/visualization/histogram_section.py index d5d7b7aa..2ec5d348 100644 --- a/popmon/visualization/histogram_section.py +++ b/popmon/visualization/histogram_section.py @@ -32,6 +32,7 @@ ) from ..base import Module from ..config import get_stat_description +from ..utils import short_date from ..visualization.utils import plot_overlay_1d_histogram_b64 @@ -83,9 +84,6 @@ def transform(self, datastore): self.logger.info(f'Generating section "{self.section_name}".') - def short_date(date): - return date if len(date) <= 22 else date[:22] - for feature in tqdm(features, ncols=100): df = data_obj.get(feature, pd.DataFrame()) diff --git a/popmon/visualization/section_generator.py b/popmon/visualization/section_generator.py index 55bf2603..eae583cb 100644 --- a/popmon/visualization/section_generator.py +++ b/popmon/visualization/section_generator.py @@ -28,6 +28,7 @@ from ..base import Module from ..config import get_stat_description +from ..utils import filter_metrics, short_date from ..visualization.utils import _prune, plot_bars_b64 @@ -112,9 +113,6 @@ def transform(self, datastore): f'Generating section "{self.section_name}". skip empty plots: {self.skip_empty_plots}' ) - def short_date(date): - return date if len(date) <= 22 else date[:22] - for feature in tqdm(features, ncols=100): df = data_obj.get(feature, pd.DataFrame()) fdbounds = dynamic_bounds.get(feature, pd.DataFrame(index=df.index)) @@ -129,18 +127,10 @@ def short_date(date): ) dates = [short_date(str(date)) for date in df.index.tolist()] - # get base64 encoded plot for each metric; do parallel processing to speed up. - metrics = [ - m - for m in df.columns - if not any([m.endswith(s) for s in self.ignore_stat_endswith]) - ] - if self.show_stats is not None: - metrics = [ - m - for m in metrics - if any(fnmatch.fnmatch(m, pattern) for pattern in self.show_stats) - ] + metrics = filter_metrics( + df.columns, self.ignore_stat_endswith, self.show_stats + ) + plots = Parallel(n_jobs=num_cores)( delayed(_plot_metric)( feature, diff --git a/popmon/visualization/traffic_light_section_generator.py b/popmon/visualization/traffic_light_section_generator.py index 184b4161..ab11e584 100644 --- a/popmon/visualization/traffic_light_section_generator.py +++ b/popmon/visualization/traffic_light_section_generator.py @@ -18,7 +18,6 @@ # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -import fnmatch import multiprocessing import numpy as np @@ -28,6 +27,7 @@ from ..base import Module from ..config import get_stat_description +from ..utils import filter_metrics, short_date from ..visualization.utils import ( _prune, plot_traffic_lights_alerts_b64, @@ -120,9 +120,6 @@ def transform(self, datastore): f'Generating section "{self.section_name}". skip empty plots: {self.skip_empty_plots}' ) - def short_date(date): - return date if len(date) <= 22 else date[:22] - for feature in tqdm(features, ncols=100): df = data_obj.get(feature, pd.DataFrame()) fdbounds = dynamic_bounds.get(feature, pd.DataFrame(index=df.index)) @@ -137,18 +134,9 @@ def short_date(date): ) dates = [short_date(str(date)) for date in df.index.tolist()] - # get base64 encoded plot for each metric; do parallel processing to speed up. - metrics = [ - m - for m in df.columns - if not any([m.endswith(s) for s in self.ignore_stat_endswith]) - ] - if self.show_stats is not None: - metrics = [ - m - for m in metrics - if any(fnmatch.fnmatch(m, pattern) for pattern in self.show_stats) - ] + metrics = filter_metrics( + df.columns, self.ignore_stat_endswith, self.show_stats + ) plots = [] if self.plot_overview: From f61fbdbfa421cfdf4572e549078a4d771e79710e Mon Sep 17 00:00:00 2001 From: sbrugman Date: Fri, 16 Apr 2021 19:09:02 +0200 Subject: [PATCH 11/30] refactor: replace `dict()` with `{}` outside src --- setup.py | 6 +++--- tests/popmon/alerting/test_apply_tl_bounds.py | 9 +++----- .../analysis/profiling/test_apply_func.py | 21 +++++++------------ tests/popmon/conftest.py | 3 +-- tests/popmon/io/test_file_writer.py | 4 ++-- 5 files changed, 16 insertions(+), 27 deletions(-) diff --git a/setup.py b/setup.py index 4310a56a..5b89ad30 100644 --- a/setup.py +++ b/setup.py @@ -80,8 +80,8 @@ def setup_package() -> None: ], # files to be shipped with the installation, under: popmon/popmon/ # after installation, these can be found with the functions in resources.py - package_data=dict( - popmon=[ + package_data={ + "popmon": [ "visualization/templates/*.html", "visualization/templates/assets/css/*.css", "visualization/templates/assets/js/*.js", @@ -89,7 +89,7 @@ def setup_package() -> None: "test_data/*.json*", "notebooks/popmon*tutorial*.ipynb", ] - ), + }, entry_points={ "console_scripts": ["popmon_run = popmon.pipeline.amazing_pipeline:run"] }, diff --git a/tests/popmon/alerting/test_apply_tl_bounds.py b/tests/popmon/alerting/test_apply_tl_bounds.py index 9bf01986..d2021aa4 100644 --- a/tests/popmon/alerting/test_apply_tl_bounds.py +++ b/tests/popmon/alerting/test_apply_tl_bounds.py @@ -71,8 +71,7 @@ def test_apply_monitoring_business_rules(): def test_apply_dynamic_traffic_light_bounds(): - datastore = dict() - datastore["to_profile"] = {"asc_numbers": get_test_data()} + datastore = {"to_profile": {"asc_numbers": get_test_data()}} conf = {"monitoring_rules": {"*_pull": [7, 4, -4, -7]}} @@ -133,8 +132,7 @@ def test_apply_dynamic_traffic_light_bounds(): def test_apply_static_traffic_light_bounds(): - datastore = dict() - datastore["to_profile"] = {"asc_numbers": get_test_data()} + datastore = {"to_profile": {"asc_numbers": get_test_data()}} conf = {"monitoring_rules": {"*_pull": [7, 4, -4, -7]}} @@ -226,8 +224,7 @@ def test_rolling_window_funcs(): def test_report_traffic_light_bounds(): - datastore = dict() - datastore["to_profile"] = {"asc_numbers": get_test_data()} + datastore = {"to_profile": {"asc_numbers": get_test_data()}} conf = { "monitoring_rules": { diff --git a/tests/popmon/analysis/profiling/test_apply_func.py b/tests/popmon/analysis/profiling/test_apply_func.py index 7820f3cf..8a53e87e 100644 --- a/tests/popmon/analysis/profiling/test_apply_func.py +++ b/tests/popmon/analysis/profiling/test_apply_func.py @@ -21,8 +21,7 @@ def get_test_data(): def test_pull(): - datastore = dict() - datastore["to_profile"] = {"asc_numbers": get_test_data()} + datastore = {"to_profile": {"asc_numbers": get_test_data()}} module1 = ApplyFunc(apply_to_key="to_profile") module1.add_apply_func(np.std, suffix="_std", entire=True) @@ -48,8 +47,7 @@ def test_pull(): def test_apply_func_module(): - datastore = dict() - datastore["to_profile"] = {"asc_numbers": get_test_data()} + datastore = {"to_profile": {"asc_numbers": get_test_data()}} def func(x): return x + 1 @@ -73,8 +71,7 @@ def func(x): def test_variance_comparer(): - datastore = dict() - datastore["to_profile"] = test_comparer_df + datastore = {"to_profile": test_comparer_df} module1 = ApplyFunc( apply_to_key="to_profile", features=["the_feature", "dummy_feature"] @@ -101,8 +98,7 @@ def test_variance_comparer(): def test_reference_pull_comparer(): - datastore = dict() - datastore["to_profile"] = test_comparer_df + datastore = {"to_profile": test_comparer_df} mod = ReferencePullCalculator( reference_key="to_profile", @@ -120,8 +116,7 @@ def test_reference_pull_comparer(): def test_median_mad_pull_comparer(): - datastore = dict() - datastore["to_profile"] = test_comparer_df + datastore = {"to_profile": test_comparer_df} mod = RefMedianMadPullCalculator( reference_key="to_profile", @@ -139,8 +134,7 @@ def test_median_mad_pull_comparer(): def test_rolling_pull_comparer(): - datastore = dict() - datastore["to_profile"] = test_comparer_df + datastore = {"to_profile": test_comparer_df} mod = RollingPullCalculator( read_key="to_profile", features=["the_feature", "dummy_feature"], window=3 @@ -157,8 +151,7 @@ def test_rolling_pull_comparer(): def test_expanding_pull_comparer(): - datastore = dict() - datastore["to_profile"] = test_comparer_df + datastore = {"to_profile": test_comparer_df} mod = ExpandingPullCalculator( read_key="to_profile", features=["the_feature", "dummy_feature"] diff --git a/tests/popmon/conftest.py b/tests/popmon/conftest.py index 884d7df6..6f020a27 100644 --- a/tests/popmon/conftest.py +++ b/tests/popmon/conftest.py @@ -9,8 +9,7 @@ def get_comparer_data(): - - test_comparer_df = dict() + test_comparer_df = {} df = pd.DataFrame( data={ "mae": [0.1, 0.11, 0.12, 0.2, 0.09], diff --git a/tests/popmon/io/test_file_writer.py b/tests/popmon/io/test_file_writer.py index 0d043d8d..c00fa308 100644 --- a/tests/popmon/io/test_file_writer.py +++ b/tests/popmon/io/test_file_writer.py @@ -10,7 +10,7 @@ def get_ready_ds(): - return copy.deepcopy(dict(my_data=DATA)) + return copy.deepcopy({"my_data": DATA}) def to_json(data, **kwargs): @@ -36,7 +36,7 @@ def test_file_writer_json_with_kwargument(): def test_file_writer_not_a_func(): datastore = get_ready_ds() with pytest.raises(TypeError): - FileWriter("my_data", apply_func=dict()).transform(datastore) + FileWriter("my_data", apply_func={}).transform(datastore) def test_file_writer_df(): From 7e0d479f9a550f2ec160699dfccd90aa78064340 Mon Sep 17 00:00:00 2001 From: sbrugman Date: Thu, 15 Apr 2021 01:16:35 +0200 Subject: [PATCH 12/30] feat: tabular traffic light/alerts overviews --- popmon/resources.py | 1 + .../visualization/alert_section_generator.py | 1 + .../templates/assets/css/custom-style.css | 35 +++++ popmon/visualization/templates/card.html | 9 +- popmon/visualization/templates/table.html | 36 +++++ .../traffic_light_section_generator.py | 18 ++- popmon/visualization/utils.py | 127 +++++++----------- 7 files changed, 145 insertions(+), 82 deletions(-) create mode 100644 popmon/visualization/templates/table.html diff --git a/popmon/resources.py b/popmon/resources.py index c15b996e..5a1d4d86 100644 --- a/popmon/resources.py +++ b/popmon/resources.py @@ -50,6 +50,7 @@ resource_filename(popmon.__name__, "visualization/templates") ) ) +_TEMPLATES_ENV.filters["fmt_metric"] = lambda x: x.replace("_", " ") def _resource(resource_type, name: str) -> str: diff --git a/popmon/visualization/alert_section_generator.py b/popmon/visualization/alert_section_generator.py index c8111def..e0251478 100644 --- a/popmon/visualization/alert_section_generator.py +++ b/popmon/visualization/alert_section_generator.py @@ -137,6 +137,7 @@ def transform(self, datastore): if self.plot_overview: plots.append( _plot_metrics( + feature, [m for m in metrics if not m.endswith("worst")], dates, df, diff --git a/popmon/visualization/templates/assets/css/custom-style.css b/popmon/visualization/templates/assets/css/custom-style.css index 381e1e8d..c41fed58 100644 --- a/popmon/visualization/templates/assets/css/custom-style.css +++ b/popmon/visualization/templates/assets/css/custom-style.css @@ -53,4 +53,39 @@ section h2 { .card-image-top { padding-bottom: 7px; +} + +/* overview tables */ +table.overview{ + margin: 25px; +} +table.overview tbody td.metric{ + white-space: nowrap; +} +table.overview tbody td.cell{ + border: 1px solid #333333; + text-align: center; +} +table.overview td.cell-green{ + background: green; +} +table.overview td.cell-yellow{ + background: yellow; +} +table.overview td.cell-red{ + background: red; +} +table.overview tfoot td { + padding-top: 5px; + text-align: center; +} +table.overview tfoot td span{ + -ms-writing-mode: tb-rl; + -webkit-writing-mode: vertical-rl; + writing-mode: vertical-rl; + transform: rotate(180deg); + white-space: nowrap; + + font-size: 14px; + font-weight: 300; } \ No newline at end of file diff --git a/popmon/visualization/templates/card.html b/popmon/visualization/templates/card.html index 143f016d..14eb95ce 100644 --- a/popmon/visualization/templates/card.html +++ b/popmon/visualization/templates/card.html @@ -1,11 +1,16 @@
+
-

{{metric.name}}

+

{{metric.name | fmt_metric}}

{% if metric.description|length %}

{{metric.description}}

{% endif %}
- + {% if 'table' in metric.plot %} + {{ metric.plot }} + {% else %} + + {% endif %}
\ No newline at end of file diff --git a/popmon/visualization/templates/table.html b/popmon/visualization/templates/table.html new file mode 100644 index 00000000..6699fb89 --- /dev/null +++ b/popmon/visualization/templates/table.html @@ -0,0 +1,36 @@ + + +{% for metric in metrics %} + + + {% for label in labels %} + {% if data[metric][label] | length == 1 %} + {% with class = data[metric][label][0] %} + + {% endwith %} + {% elif data[metric][label] | length == 2 %} + {% with rgba, value = data[metric][label] %} + + {% endwith %} + {% endif %} + {% endfor %} + +{% endfor %} + + + + + {% for label in labels %} + + {% endfor %} + + +
+ {% if links %} + + {% endif %} + {{ metric | fmt_metric }} + {% if links %} + + {% endif %} +  {{ value }}
 {{ label }}
\ No newline at end of file diff --git a/popmon/visualization/traffic_light_section_generator.py b/popmon/visualization/traffic_light_section_generator.py index ab11e584..503fd027 100644 --- a/popmon/visualization/traffic_light_section_generator.py +++ b/popmon/visualization/traffic_light_section_generator.py @@ -32,7 +32,7 @@ _prune, plot_traffic_lights_alerts_b64, plot_traffic_lights_b64, - plot_traffic_lights_heatmap_b64, + plot_traffic_lights_overview, ) @@ -142,6 +142,7 @@ def transform(self, datastore): if self.plot_overview: plots.append( _plot_metrics( + feature, metrics, dates, df, @@ -203,7 +204,15 @@ def _plot_metric(metric, dates, values, last_n, skip_first_n, skip_last_n, skip_ def _plot_metrics( - metrics, dates, df, last_n, skip_first_n, skip_last_n, skip_empty, style="heatmap" + feature, + metrics, + dates, + df, + last_n, + skip_first_n, + skip_last_n, + skip_empty, + style="heatmap", ): # prune dates and values dates = _prune(dates, last_n, skip_first_n, skip_last_n) @@ -222,11 +231,12 @@ def _plot_metrics( # make plot. note: slow! if style == "heatmap": - plot = plot_traffic_lights_heatmap_b64( - values, metrics=nonempty_metrics, labels=dates + plot = plot_traffic_lights_overview( + feature, values, metrics=nonempty_metrics, labels=dates ) elif style == "alerts": plot = plot_traffic_lights_alerts_b64( + feature, values, metrics=nonempty_metrics, labels=dates, diff --git a/popmon/visualization/utils.py b/popmon/visualization/utils.py index 30cdb89e..0cee1363 100644 --- a/popmon/visualization/utils.py +++ b/popmon/visualization/utils.py @@ -21,13 +21,15 @@ import logging import math from io import BytesIO, StringIO +from typing import List import numpy as np import pandas as pd import pybase64 from ing_theme_matplotlib import mpl_style from matplotlib import pyplot as plt -from matplotlib.colors import BoundaryNorm, ListedColormap + +from popmon.resources import templates_env NUM_NS_DAY = 24 * 3600 * int(1e9) @@ -153,38 +155,55 @@ def plot_bars_b64(data, labels=None, bounds=None, ylim=False, skip_empty=True): return plt_to_str() -def plot_traffic_lights_heatmap_b64(data, metrics=None, labels=None): - fig, ax = plt.subplots(figsize=(14, 4.5)) - - cm = ListedColormap(["green", "yellow", "red"]) - norm = BoundaryNorm([0, 1, 2], 2) - - _ = ax.imshow(data, interpolation="none", aspect="equal", cmap=cm, norm=norm) - - # Major ticks - ax.set_xticks(np.arange(0, len(labels), 1)) - ax.set_yticks(np.arange(0, len(metrics), 1)) - - # Labels for major ticks - ax.set_xticklabels(labels) - ax.set_yticklabels(metrics) - - # Minor ticks - ax.set_xticks(np.arange(-0.50, len(labels), 1), minor=True) - ax.set_yticks(np.arange(-0.50, len(metrics), 1), minor=True) - - plt.setp(ax.get_xticklabels(), rotation=90, ha="right", rotation_mode="anchor") - - # Gridlines based on minor ticks - ax.grid(False) - ax.grid(which="minor", color="#333333", linestyle="-", linewidth=1, alpha=1) - - fig.tight_layout() +def render_traffic_lights_table(feature, data, metrics: List[str], labels: List[str]): + colors = {} + color_map = ["green", "yellow", "red"] + for c1, metric in enumerate(metrics): + colors[metric] = {} + for c2, label in enumerate(labels): + colors[metric][label] = [color_map[data[c1][c2]]] + + return templates_env( + "table.html", + feature=feature, + metrics=metrics, + labels=labels, + data=colors, + links=True, + ) + + +def plot_traffic_lights_overview(feature, data, metrics=None, labels=None): + return render_traffic_lights_table(feature, data, metrics, labels) + + +def render_alert_aggregate_table(feature, data, metrics: List[str], labels: List[str]): + colors = {} + for c1, metric in enumerate(metrics): + colors[metric] = {} + row_max = np.max(data[c1]) + for c2, label in enumerate(labels): + a = data[c1][c2] / row_max + if metric.endswith("green"): + rgba = (0, 128, 0, a) + elif metric.endswith("yellow"): + rgba = (255, 255, 0, a) + else: + rgba = (255, 0, 0, a) + rgba = (str(v) for v in rgba) + colors[metric][label] = (rgba, data[c1][c2]) - return plt_to_str() + return templates_env( + "table.html", + feature=feature, + metrics=metrics, + labels=labels, + data=colors, + links=False, + ) -def plot_traffic_lights_alerts_b64(data, metrics=None, labels=None): +def plot_traffic_lights_alerts_b64(feature, data, metrics=None, labels=None): assert data.shape[0] == 3 # Reorder metrics if needed @@ -195,53 +214,9 @@ def plot_traffic_lights_alerts_b64(data, metrics=None, labels=None): if [pos_green, pos_yellow, pos_red] != [0, 1, 2]: data[[0, 1, 2]] = data[[pos_green, pos_yellow, pos_red]] - metrics = ["n_green", "n_yellow", "n_red"] - - fig, ax = plt.subplots(figsize=(14, 4.5)) - - N = 256 - yellow = np.ones((N, 4)) - yellow[:, 0] = np.linspace(1, 255 / 256, N) - yellow[:, 1] = np.linspace(1, 232 / 256, N) - yellow[:, 2] = np.linspace(1, 11 / 256, N) - yellow_cmp = ListedColormap(yellow) - - cmaps = reversed(["Reds", yellow_cmp, "Greens"]) - # https://stackoverflow.com/questions/60325792/seaborn-heatmap-color-by-row - for idx, cmap in enumerate(cmaps): - _ = ax.imshow( - np.vstack([data[idx, :], data[idx, :]]), - aspect="equal", - cmap=cmap, - extent=[-0.5, data.shape[1] - 0.5, idx - 0.5, idx + 0.5], - ) - - # Major ticks - ax.set_xticks(np.arange(0, len(labels), 1)) - ax.set_yticks(np.arange(0, len(metrics), 1)) - - # Labels for major ticks - ax.set_xticklabels(labels) - ax.set_yticklabels(metrics) - - # Minor ticks - ax.set_xticks(np.arange(-0.50, len(labels), 1), minor=True) - ax.set_yticks(np.arange(-0.50, len(metrics), 1), minor=True) - - plt.setp(ax.get_xticklabels(), rotation=90, ha="right", rotation_mode="anchor") + metrics = ["# green", "# yellow", "# red"] - # Annotations - for i in range(len(metrics)): - for j in range(len(labels)): - ax.text(j, i, f"{data[i, j]:.0f}", ha="center", va="center", color="black") - - # Gridlines based on minor ticks - ax.grid(False) - ax.grid(which="minor", color="#333333", linestyle="-", linewidth=1, alpha=1) - - fig.tight_layout() - - return plt_to_str() + return render_alert_aggregate_table(feature, data.astype(int), metrics, labels) def plot_traffic_lights_b64(data, labels=None, skip_empty=True): From 47c41265aa94678205fd577ffed28f98d71139c0 Mon Sep 17 00:00:00 2001 From: sbrugman Date: Tue, 20 Apr 2021 19:08:05 +0200 Subject: [PATCH 13/30] build: pre-commit update --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b5051e13..4d25dc83 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,8 +9,8 @@ repos: - id: isort files: '.*' args: [ --profile=black, --project=popmon, --thirdparty histogrammar, --thirdparty pybase64 ] -- repo: https://gitlab.com/pycqa/flake8 - rev: "3.9.0" +- repo: https://github.com/PyCQA/flake8 + rev: "3.9.1" hooks: - id: flake8 args: [ "--select=E9,F63,F7,F82"] From af2a5e79cad1fe51282b07488267994e3cab6b59 Mon Sep 17 00:00:00 2001 From: sbrugman Date: Tue, 20 Apr 2021 19:08:38 +0200 Subject: [PATCH 14/30] style: build.yml name missing step --- .github/workflows/build.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6fa4e2b4..57574f5f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,6 +1,3 @@ -# This workflow will install Python dependencies, run tests and lint with a single version of Python -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - name: build on: @@ -62,18 +59,16 @@ jobs: run: | python -m pip install --upgrade pip pip install -e . - - name: + - name: Build examples run: | cd examples python synthetic_data.py python flight_delays.py - - uses: actions/upload-artifact@v2 with: name: synthetic-report path: examples/test_data_report.html if-no-files-found: error - - uses: actions/upload-artifact@v2 with: name: flight-delays-report From dee8a21a5411a35afd2e897d8dfb1d434bce8c9e Mon Sep 17 00:00:00 2001 From: sbrugman Date: Wed, 21 Apr 2021 18:18:25 +0200 Subject: [PATCH 15/30] docs: fix typo --- popmon/analysis/apply_func.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/popmon/analysis/apply_func.py b/popmon/analysis/apply_func.py index af70cbcc..9dcb402f 100644 --- a/popmon/analysis/apply_func.py +++ b/popmon/analysis/apply_func.py @@ -58,7 +58,7 @@ def __init__( :param list apply_funcs: functions to apply (list of dicts): - 'func': function to apply - - 'suffix' (string, optional): suffix added to each metric. default is fuction name. + - 'suffix' (string, optional): suffix added to each metric. default is function name. - 'prefix' (string, optional): prefix added to each metric. - 'features' (list, optional): features the function is applied to. Overwrites features above - 'metrics' (list, optional): metrics the function is applied to. Overwrites metrics above From a5c4cae8699696da47ac3f8cdad1ef3add8783dd Mon Sep 17 00:00:00 2001 From: sbrugman Date: Mon, 26 Apr 2021 20:16:32 +0200 Subject: [PATCH 16/30] refactor: convert dict constructor to dict literal --- tests/popmon/alerting/test_apply_tl_bounds.py | 2 +- tests/popmon/alerting/test_integration.py | 4 +- .../analysis/comparison/test_hist_comparer.py | 28 +++--- tests/popmon/analysis/test_functions.py | 96 +++++++++---------- 4 files changed, 65 insertions(+), 65 deletions(-) diff --git a/tests/popmon/alerting/test_apply_tl_bounds.py b/tests/popmon/alerting/test_apply_tl_bounds.py index d2021aa4..e89dd412 100644 --- a/tests/popmon/alerting/test_apply_tl_bounds.py +++ b/tests/popmon/alerting/test_apply_tl_bounds.py @@ -193,7 +193,7 @@ def test_apply_static_traffic_light_bounds(): def test_rolling_window_funcs(): - datastore = dict(to_profile={"asc_numbers": get_test_data()}) + datastore = {"to_profile": {"asc_numbers": get_test_data()}} m = ApplyFunc( apply_to_key="to_profile", features=["asc_numbers"], metrics=["a", "b"] diff --git a/tests/popmon/alerting/test_integration.py b/tests/popmon/alerting/test_integration.py index 701fd96b..a5446c84 100644 --- a/tests/popmon/alerting/test_integration.py +++ b/tests/popmon/alerting/test_integration.py @@ -79,7 +79,7 @@ def test_traffic_light_summary(): tls = ApplyFunc( apply_to_key="output_data", - apply_funcs=[dict(func=traffic_light_summary, axis=1, suffix="")], + apply_funcs=[{"func": traffic_light_summary, "axis": 1, "suffix": ""}], assign_to_key="alerts", ) @@ -124,7 +124,7 @@ def test_traffic_light_summary_combination(): tls = ApplyFunc( apply_to_key="output_data", - apply_funcs=[dict(func=traffic_light_summary, axis=1, suffix="")], + apply_funcs=[{"func": traffic_light_summary, "axis": 1, "suffix": ""}], assign_to_key="alerts", ) diff --git a/tests/popmon/analysis/comparison/test_hist_comparer.py b/tests/popmon/analysis/comparison/test_hist_comparer.py index 256a2f99..f3ee18c1 100644 --- a/tests/popmon/analysis/comparison/test_hist_comparer.py +++ b/tests/popmon/analysis/comparison/test_hist_comparer.py @@ -39,26 +39,26 @@ def test_hist_compare(): ApplyFunc( apply_to_key="output_hist", apply_funcs=[ - dict( - func=expanding_hist, - shift=1, - suffix="sum", - entire=True, - hist_name="histogram", - ) + { + "func": expanding_hist, + "shift": 1, + "suffix": "sum", + "entire": True, + "hist_name": "histogram", + } ], ), ApplyFunc( apply_to_key="output_hist", assign_to_key="comparison", apply_funcs=[ - dict( - func=hist_compare, - hist_name1="histogram", - hist_name2="histogram_sum", - suffix="", - axis=1, - ) + { + "func": hist_compare, + "hist_name1": "histogram", + "hist_name2": "histogram_sum", + "suffix": "", + "axis": 1, + } ], ), ] diff --git a/tests/popmon/analysis/test_functions.py b/tests/popmon/analysis/test_functions.py index 71ff0b7d..7640228a 100644 --- a/tests/popmon/analysis/test_functions.py +++ b/tests/popmon/analysis/test_functions.py @@ -63,13 +63,13 @@ def test_expanding_hist(): ApplyFunc( apply_to_key="output_hist", apply_funcs=[ - dict( - func=expanding_hist, - shift=1, - suffix="sum", - entire=True, - hist_name="histogram", - ) + { + "func": expanding_hist, + "shift": 1, + "suffix": "sum", + "entire": True, + "hist_name": "histogram", + } ], ), ] @@ -222,14 +222,14 @@ def test_rolling_hist(): ApplyFunc( apply_to_key="output_hist", apply_funcs=[ - dict( - func=rolling_hist, - window=5, - shift=1, - suffix="sum", - entire=True, - hist_name="histogram", - ) + { + "func": rolling_hist, + "window": 5, + "shift": 1, + "suffix": "sum", + "entire": True, + "hist_name": "histogram", + } ], ), ] @@ -333,7 +333,7 @@ def test_normalized_hist_mean_cov(): ApplyFunc( apply_to_key="output_hist", assign_to_key="output_hist", - apply_funcs=[dict(func=normalized_hist_mean_cov, suffix="")], + apply_funcs=[{"func": normalized_hist_mean_cov, "suffix": ""}], ), ] ) @@ -394,14 +394,14 @@ def test_roll_norm_hist_mean_cov(): ApplyFunc( apply_to_key="output_hist", apply_funcs=[ - dict( - func=roll_norm_hist_mean_cov, - hist_name="histogram", - window=5, - shift=1, - suffix="", - entire=True, - ) + { + "func": roll_norm_hist_mean_cov, + "hist_name": "histogram", + "window": 5, + "shift": 1, + "suffix": "", + "entire": True, + } ], ), ] @@ -514,13 +514,13 @@ def test_expand_norm_hist_mean_cov(): ApplyFunc( apply_to_key="output_hist", apply_funcs=[ - dict( - func=expand_norm_hist_mean_cov, - hist_name="histogram", - shift=1, - suffix="", - entire=True, - ) + { + "func": expand_norm_hist_mean_cov, + "hist_name": "histogram", + "shift": 1, + "suffix": "", + "entire": True, + } ], ), ] @@ -636,19 +636,19 @@ def test_chi_squared1(): ApplyFunc( apply_to_key="output_hist", apply_funcs=[ - dict( - func=roll_norm_hist_mean_cov, - hist_name="histogram", - window=5, - shift=1, - suffix="", - entire=True, - ) + { + "func": roll_norm_hist_mean_cov, + "hist_name": "histogram", + "window": 5, + "shift": 1, + "suffix": "", + "entire": True, + } ], ), ApplyFunc( apply_to_key="output_hist", - apply_funcs=[dict(func=relative_chi_squared, suffix="", axis=1)], + apply_funcs=[{"func": relative_chi_squared, "suffix": "", "axis": 1}], ), ] ) @@ -698,18 +698,18 @@ def test_chi_squared2(): ApplyFunc( apply_to_key="output_hist", apply_funcs=[ - dict( - func=expand_norm_hist_mean_cov, - hist_name="histogram", - shift=1, - suffix="", - entire=True, - ) + { + "func": expand_norm_hist_mean_cov, + "hist_name": "histogram", + "shift": 1, + "suffix": "", + "entire": True, + } ], ), ApplyFunc( apply_to_key="output_hist", - apply_funcs=[dict(func=relative_chi_squared, suffix="", axis=1)], + apply_funcs=[{"func": relative_chi_squared, "suffix": "", "axis": 1}], ), ] ) From 3a8be3171fa519e732fd56d7ada523cd77100a1c Mon Sep 17 00:00:00 2001 From: sbrugman Date: Mon, 26 Apr 2021 20:18:04 +0200 Subject: [PATCH 17/30] ci: bump + flake8 + isort args - bump black, pyupgrade - flake8 comprehensions - remove isort args that are no longer needed --- .pre-commit-config.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4d25dc83..254c08dd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/psf/black - rev: 20.8b1 + rev: 21.4b0 hooks: - id: black - repo: https://github.com/pycqa/isort @@ -8,14 +8,16 @@ repos: hooks: - id: isort files: '.*' - args: [ --profile=black, --project=popmon, --thirdparty histogrammar, --thirdparty pybase64 ] + args: [ --profile=black, --project=popmon ] - repo: https://github.com/PyCQA/flake8 rev: "3.9.1" hooks: - id: flake8 - args: [ "--select=E9,F63,F7,F82"] + additional_dependencies: + - flake8-comprehensions + args: [ "--select=E9,F63,F7,F82,C4"] - repo: https://github.com/asottile/pyupgrade - rev: v2.12.0 + rev: v2.13.0 hooks: - id: pyupgrade args: ['--py36-plus','--exit-zero-even-if-changed'] \ No newline at end of file From 06dcdd9f1bbc057a624603c8644758559cbbdacf Mon Sep 17 00:00:00 2001 From: sbrugman Date: Tue, 20 Apr 2021 19:20:44 +0200 Subject: [PATCH 18/30] test: add filter_metrics test for TL/default stats --- tests/popmon/test_utils.py | 47 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 tests/popmon/test_utils.py diff --git a/tests/popmon/test_utils.py b/tests/popmon/test_utils.py new file mode 100644 index 00000000..460ff1ea --- /dev/null +++ b/tests/popmon/test_utils.py @@ -0,0 +1,47 @@ +from popmon.config import config +from popmon.utils import filter_metrics + + +def test_filter_metrics(): + metrics = [ + "distinct_pull", + "filled_pull", + "nan_pull", + "mean_pull", + "std_pull", + "p05_pull", + "p10_pull", + "p50_pull", + "p85_pull", + "p95_pull", + "max_pull", + "min_pull", + "fraction_true_trend10_zscore", + "ref_unknown_labels", + "prev1_ks_zscore", + "worst", + "ref_max_prob_diff", + ] + expected = [ + "distinct_pull", + "filled_pull", + "nan_pull", + "mean_pull", + "std_pull", + "p05_pull", + "p50_pull", + "p95_pull", + "max_pull", + "min_pull", + "fraction_true_trend10_zscore", + "ref_unknown_labels", + "prev1_ks_zscore", + "worst", + "ref_max_prob_diff", + ] + assert ( + filter_metrics( + metrics, ignore_stat_endswith=[], show_stats=config["limited_stats"] + ) + == expected + ) From 494fc568819f93956e95672797b1409a2e8a19cc Mon Sep 17 00:00:00 2001 From: sbrugman Date: Tue, 20 Apr 2021 19:21:34 +0200 Subject: [PATCH 19/30] fix: extend TL config --- popmon/config.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/popmon/config.py b/popmon/config.py index 4e585dce..362b4ae4 100644 --- a/popmon/config.py +++ b/popmon/config.py @@ -95,11 +95,11 @@ "min*", "fraction_true*", "phik*", - "*unknown_labels", - "*chi2_norm", - "*ks", - "*max_prob_diff", - "*zscore", + "*unknown_labels*", + "*chi2_norm*", + "*ks*", + "*max_prob_diff*", + "*zscore*", "n_*", "worst", ], From 36120a94e394025f09ad6eec44d93ea12b02eb92 Mon Sep 17 00:00:00 2001 From: sbrugman Date: Tue, 20 Apr 2021 19:23:56 +0200 Subject: [PATCH 20/30] docs: add explanation for hiding green TLs by default --- popmon/config.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/popmon/config.py b/popmon/config.py index 362b4ae4..b25b4200 100644 --- a/popmon/config.py +++ b/popmon/config.py @@ -74,8 +74,7 @@ is represented by one bin). The yellow and red lines represent the corresponding traffic light bounds (default: 4 and 7 standard deviations with respect to the reference data).""", "comparisons": "Statistical comparisons of each time period (one bin) to the reference data.", - "traffic_lights": """Traffic light calculation for different statistics (based on the calculated - normalized residual, a.k.a. pull) for each time period.""", + "traffic_lights": "Traffic light calculation for different statistics (based on the calculated normalized residual, a.k.a. pull). Statistics for which all traffic lights are green are hidden from view by default.", "alerts": "Alerts aggregated by all traffic lights for each feature.", "histograms": "Histograms of the last few time slots (default: 2).", } From 7dc61dbec6c9fc97ef009c3f194bc7110b717ca2 Mon Sep 17 00:00:00 2001 From: sbrugman Date: Fri, 30 Apr 2021 16:22:31 +0200 Subject: [PATCH 21/30] ci: openjdk-8 requires apt-get update --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 57574f5f..8e37a7e5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -36,6 +36,7 @@ jobs: SPARK_HOME: "/home/runner/work/spark/" #${{ github.workspace }}/spark/ SPARK_LOCAL_IP: "localhost" run: | + sudo apt-get update sudo apt-get -y install openjdk-8-jdk curl https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz --output ${BUILD_DIR}/spark.tgz tar -xvzf ${BUILD_DIR}/spark.tgz && mv spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION} ${SPARK_HOME} From 0b9b32377d6dac0c8efccdbaef7c057992fd1596 Mon Sep 17 00:00:00 2001 From: sbrugman Date: Fri, 30 Apr 2021 17:29:03 +0200 Subject: [PATCH 22/30] chore: format CHANGES.rst --- CHANGES.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index dd5e7ee2..73255ca4 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,20 +2,20 @@ Release notes ============= -Version 0.4.0, (16-04-2021) ---------------------------- +Version 0.4.0 (16-04-2021) +-------------------------- Documentation: -* **docs**: include BDTWS presentation -* **docs**: clarify that ``time_axis`` should be date or numeric -* **docs**: initialize spark with both histogrammar jar files +* Include BDTWS presentation +* Clarify that ``time_axis`` should be date or numeric +* Initialize spark with both histogrammar jar files Build system -* **build**: Migrate to version 1.0.25 of ``histogrammar``. -* **build**: update ``pyupgrade`` to v2.12.0 -* **build**: update ``isort`` to 5.8.0 -* **build**: update ``flake8`` to 3.9.0 +* Migrate to version 1.0.25 of ``histogrammar``. +* Update ``pyupgrade`` to v2.12.0 +* Update ``isort`` to 5.8.0 +* Update ``flake8`` to 3.9.0 Version 0.3.14, Feb 2021 ------------------------ From f026069886a899097621454b8bd9dab8f1f6dcfa Mon Sep 17 00:00:00 2001 From: sbrugman Date: Fri, 30 Apr 2021 17:00:33 +0200 Subject: [PATCH 23/30] ci: dependabot Github-native setup --- .github/dependabot.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..98ad7289 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,18 @@ +version: 2 +updates: +- package-ecosystem: pip + directory: "/" + schedule: + interval: daily + time: "04:00" + open-pull-requests-limit: 10 + target-branch: develop + allow: + - dependency-type: direct + - dependency-type: indirect + ignore: + - dependency-name: histogrammar + versions: + - 1.0.20 + - 1.0.21 + - 1.0.23 \ No newline at end of file From a632a3544b2433f15bcaf19f8d661910947ea9db Mon Sep 17 00:00:00 2001 From: Simon Brugman Date: Wed, 5 May 2021 17:34:51 +0200 Subject: [PATCH 24/30] ci: rename commit msg precommit.ci --- .pre-commit-config.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 254c08dd..f42e0be8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,4 +20,7 @@ repos: rev: v2.13.0 hooks: - id: pyupgrade - args: ['--py36-plus','--exit-zero-even-if-changed'] \ No newline at end of file + args: ['--py36-plus','--exit-zero-even-if-changed'] + +ci: + autoupdate_commit_msg: 'ci: pre-commit-config update' From c6fa32efa11f5dc261d347d6131a7995500390a6 Mon Sep 17 00:00:00 2001 From: Simon Brugman Date: Tue, 1 Jun 2021 13:15:03 +0200 Subject: [PATCH 25/30] docs: codemotion article --- README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.rst b/README.rst index 921d16f1..ba6dd2c6 100644 --- a/README.rst +++ b/README.rst @@ -172,6 +172,8 @@ Articles +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------+----------------+ | Title | Date | Author | +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------+----------------+ +| `Population Shift Analysis: Monitoring Data Quality with Popmon `_ | May 21, 2021 | Vito Gentile | ++---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------+----------------+ | `Popmon Open Source Package — Population Shift Monitoring Made Easy `_ | May 20, 2020 | Nicole Mpozika | +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------+----------------+ From 8b43e229581f1780cc90beee21fa589e5d0c85dd Mon Sep 17 00:00:00 2001 From: Simon Brugman Date: Mon, 21 Jun 2021 18:38:33 +0200 Subject: [PATCH 26/30] ci: python build matrix (#132) * ci: python build matrix * fix: latest ing_theme_matplotlib from source installing this extension from source until a new release is available * fix: callable test --- .github/workflows/build.yml | 35 ++++++++++++++++++++++++++++++----- popmon/analysis/apply_func.py | 2 +- requirements.txt | 2 +- 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8e37a7e5..32c97d81 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,15 +7,19 @@ on: branches: [ master, develop ] jobs: - build: - runs-on: ubuntu-latest + test: + strategy: + matrix: + os: [ubuntu-latest] + python: [3.6, 3.7, 3.8] + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 - - name: Set up Python 3.6 + - name: Set up Python ${{ matrix.python }} uses: actions/setup-python@v1 with: - python-version: 3.6 + python-version: ${{ matrix.python }} - name: Install dependencies run: | python -m pip install --upgrade pip @@ -27,6 +31,25 @@ jobs: - name: Test with pytest run: | pytest -m "not spark" + + test_spark: + strategy: + matrix: + os: [ubuntu-latest] + python: [3.6] + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e . + pip install -r requirements-test.txt - name: Install Spark env: BUILD_DIR: "/home/runner/work/" #${{ github.workspace }} @@ -49,7 +72,9 @@ jobs: examples: runs-on: ubuntu-latest - needs: build + needs: + - test + - test_spark steps: - uses: actions/checkout@v2 - name: Set up Python 3.6 diff --git a/popmon/analysis/apply_func.py b/popmon/analysis/apply_func.py index 9dcb402f..7c9e62da 100644 --- a/popmon/analysis/apply_func.py +++ b/popmon/analysis/apply_func.py @@ -106,7 +106,7 @@ def add_apply_func( :param kwargs: (dict, optional) kwargs for 'func' """ # check inputs - if not isinstance(func, collections.Callable): + if not callable(func): raise TypeError("functions in ApplyFunc must be callable objects") if suffix is not None and not isinstance(suffix, str): raise TypeError("prefix, and suffix in ApplyFunc must be strings or None.") diff --git a/requirements.txt b/requirements.txt index e0700d5a..3715373e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,4 +9,4 @@ matplotlib>=2.2.3 joblib>=0.14.0 pybase64>=1.0.1 htmlmin -ing_theme_matplotlib +ing_theme_matplotlib @ git+https://gitlab.com/ing_rpaa/ing_theme_matplotlib.git From 9fb3c0896b87bb1ecfee65448ce9d0900f13d541 Mon Sep 17 00:00:00 2001 From: sbrugman Date: Fri, 30 Apr 2021 17:26:11 +0200 Subject: [PATCH 27/30] ci: automate generation of release notes via standard-version to CHANGELOG.md. --- .versionrc.json | 16 ++++++++++++++++ CHANGELOG.md | 1 + CHANGES.rst | 2 ++ Makefile | 3 +++ make.bat | 5 +++++ package.json | 12 ++++++++++++ 6 files changed, 39 insertions(+) create mode 100644 .versionrc.json create mode 100644 CHANGELOG.md create mode 100644 package.json diff --git a/.versionrc.json b/.versionrc.json new file mode 100644 index 00000000..f5efefe3 --- /dev/null +++ b/.versionrc.json @@ -0,0 +1,16 @@ +{ + "header": "Release notes", + "types": [ + {"type": "feat", "section": "\uD83C\uDF89 Features"}, + {"type": "fix", "section": "\uD83D\uDC1B Bug fixes"}, + {"type": "perf", "section": "⚡ Performance improvements"}, + {"type": "docs", "section": "\uD83D\uDCD6 Documentation"}, + {"type": "style", "section": "\uD83D\uDC77\u200D♂️ Internal improvements"}, + {"type": "refactor", "section": "\uD83D\uDC77\u200D♂️ Internal improvements"}, + {"type": "build", "section": "⬆️ Dependencies"}, + {"type": "chore", "hidden": true}, + {"type": "test", "hidden": true} + ], + "commitUrlFormat": "https://github.com/ing-bank/popmon/commits/{{hash}}", + "compareUrlFormat": "https://github.com/ing-bank/popmon/compare/{{previousTag}}...{{currentTag}}" +} \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..d7de275f --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1 @@ +# Release notes diff --git a/CHANGES.rst b/CHANGES.rst index 73255ca4..e70e31b6 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,8 @@ Release notes ============= +The release notes for more recent versions have move to `CHANGELOG.md `_ + Version 0.4.0 (16-04-2021) -------------------------- Documentation: diff --git a/Makefile b/Makefile index e53c919f..70582b20 100644 --- a/Makefile +++ b/Makefile @@ -3,3 +3,6 @@ lint: install: pip install -e . + +changelog: + npm run release diff --git a/make.bat b/make.bat index 4520dc75..461e341c 100644 --- a/make.bat +++ b/make.bat @@ -12,5 +12,10 @@ IF "%1%" == "install" ( GOTO end ) +if "%1%" == "changelog" ( + npm run release + GOTO end +) + ECHO "No command matched" :end \ No newline at end of file diff --git a/package.json b/package.json new file mode 100644 index 00000000..a77283e6 --- /dev/null +++ b/package.json @@ -0,0 +1,12 @@ +{ + "scripts": { + "release": "standard-version" + }, + "standard-version": { + "skip": { + "bump": true, + "commit": true, + "tag": true + } + } +} \ No newline at end of file From dd7f541175202ee47cb3859f4a2b0c46f7ad6796 Mon Sep 17 00:00:00 2001 From: Simon Brugman Date: Wed, 23 Jun 2021 09:00:43 +0200 Subject: [PATCH 28/30] build: pre-commit autoupdate --- .pre-commit-config.yaml | 8 ++++---- popmon/hist/hist_utils.py | 4 ++-- popmon/stats/numpy.py | 2 +- popmon/stitching/hist_stitcher.py | 4 ++-- popmon/visualization/alert_section_generator.py | 8 +++----- popmon/visualization/section_generator.py | 8 +++----- 6 files changed, 15 insertions(+), 19 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f42e0be8..849e4d65 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,23 +1,23 @@ repos: - repo: https://github.com/psf/black - rev: 21.4b0 + rev: 21.6b0 hooks: - id: black - repo: https://github.com/pycqa/isort - rev: 5.8.0 + rev: 5.9.1 hooks: - id: isort files: '.*' args: [ --profile=black, --project=popmon ] - repo: https://github.com/PyCQA/flake8 - rev: "3.9.1" + rev: "3.9.2" hooks: - id: flake8 additional_dependencies: - flake8-comprehensions args: [ "--select=E9,F63,F7,F82,C4"] - repo: https://github.com/asottile/pyupgrade - rev: v2.13.0 + rev: v2.19.4 hooks: - id: pyupgrade args: ['--py36-plus','--exit-zero-even-if-changed'] diff --git a/popmon/hist/hist_utils.py b/popmon/hist/hist_utils.py index ae8f81b1..e46b2ba1 100644 --- a/popmon/hist/hist_utils.py +++ b/popmon/hist/hist_utils.py @@ -56,10 +56,10 @@ def sum_entries(hist, default=True): # double check number of entries, sometimes not well set if hasattr(hist, "bins"): # loop over all counters and integrate over y (=j) - return sum([sum_entries(bi) for bi in hist.bins.values()]) + return sum(sum_entries(bi) for bi in hist.bins.values()) elif hasattr(hist, "values"): # loop over all counters and integrate over y (=j) - return sum([sum_entries(bi) for bi in hist.values]) + return sum(sum_entries(bi) for bi in hist.values) elif hasattr(hist, "entries"): # only count histogrammar.Count() objects return hist.entries diff --git a/popmon/stats/numpy.py b/popmon/stats/numpy.py index 231a2fad..3df57d88 100644 --- a/popmon/stats/numpy.py +++ b/popmon/stats/numpy.py @@ -206,7 +206,7 @@ def quantile(a, q, weights=None, axis=None, keepdims: bool = False): if keepdims: shape = ( *y.shape[:-1], - *[1 if i in axis else x for i, x in enumerate(a.shape)], + *(1 if i in axis else x for i, x in enumerate(a.shape)), ) else: shape = *y.shape[:-1], *a_moved.shape[: -len(destination)] diff --git a/popmon/stitching/hist_stitcher.py b/popmon/stitching/hist_stitcher.py index 2b095f6e..4e7baec9 100644 --- a/popmon/stitching/hist_stitcher.py +++ b/popmon/stitching/hist_stitcher.py @@ -290,7 +290,7 @@ def _find_max_time_bin_index(self, hists_basis, features_basis, time_axis): max_time_bin_idx = None if all_sparse or all_cat: max_time_bin_idx = max( - [max(h.bins.keys()) for h in hist_list if len(h.bins) > 0] + max(h.bins.keys()) for h in hist_list if len(h.bins) > 0 ) return max_time_bin_idx @@ -426,7 +426,7 @@ def _stitch_by_update(self, mode, hist_list): # update bins consecutively for each time-delta. for hist in hist_list: hsum.bins.update(hist.bins) - hsum.entries = sum([b.entries for b in hsum.bins.values()]) + hsum.entries = sum(b.entries for b in hsum.bins.values()) else: for hist in hist_list: hsum += hist diff --git a/popmon/visualization/alert_section_generator.py b/popmon/visualization/alert_section_generator.py index e0251478..5d2daac6 100644 --- a/popmon/visualization/alert_section_generator.py +++ b/popmon/visualization/alert_section_generator.py @@ -209,11 +209,9 @@ def _plot_metric( # pick up dynamic traffic light boundaries names = [prefix + metric + suffix for suffix in suffices] dbounds = tuple( - [ - _prune(fdbounds[n].tolist(), last_n, skip_first_n, skip_last_n) - for n in names - if n in fdbounds.columns - ] + _prune(fdbounds[n].tolist(), last_n, skip_first_n, skip_last_n) + for n in names + if n in fdbounds.columns ) # choose dynamic bounds if present bounds = dbounds if len(dbounds) > 0 else sbounds diff --git a/popmon/visualization/section_generator.py b/popmon/visualization/section_generator.py index eae583cb..07a395ac 100644 --- a/popmon/visualization/section_generator.py +++ b/popmon/visualization/section_generator.py @@ -190,11 +190,9 @@ def _plot_metric( # pick up dynamic traffic light boundaries names = [prefix + metric + suffix for suffix in suffices] dbounds = tuple( - [ - _prune(fdbounds[n].tolist(), last_n, skip_first_n, skip_last_n) - for n in names - if n in fdbounds.columns - ] + _prune(fdbounds[n].tolist(), last_n, skip_first_n, skip_last_n) + for n in names + if n in fdbounds.columns ) # choose dynamic bounds if present bounds = dbounds if len(dbounds) > 0 else sbounds From 8be143c3b95fab370f65484faff7d924a636824c Mon Sep 17 00:00:00 2001 From: Simon Brugman Date: Wed, 23 Jun 2021 10:54:06 +0200 Subject: [PATCH 29/30] chore: version bump --- popmon/version.py | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/popmon/version.py b/popmon/version.py index a22f61e5..e01c00a1 100644 --- a/popmon/version.py +++ b/popmon/version.py @@ -1,6 +1,6 @@ """THIS FILE IS AUTO-GENERATED BY SETUP.PY.""" name = "popmon" -version = "0.4.0" -full_version = "0.4.0" +version = "0.4.1" +full_version = "0.4.1" release = True diff --git a/setup.py b/setup.py index 5b89ad30..7eb4287c 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ MAJOR = 0 REVISION = 4 -PATCH = 0 +PATCH = 1 DEV = False # NOTE: also update version at: README.rst From 5fb51bce32387cb5b6721a750cc18ceac8c66b91 Mon Sep 17 00:00:00 2001 From: Simon Brugman Date: Wed, 23 Jun 2021 10:54:19 +0200 Subject: [PATCH 30/30] chore: changelog --- CHANGELOG.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7de275f..a9f19afd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1 +1,35 @@ # Release notes +## [Version 0.4.1](https://github.com/ing-bank/popmon/compare/v0.4.0...v0.4.1) (2021-06-23) + + +### 🎉 Features + +* tabular traffic light/alerts overviews ([1d31265](https://github.com/ing-bank/popmon/commits/1d312653e0f2e788f4631839d201460f7e4ff562)) + + +### 👷‍♂️ Internal improvements + +* add utils to prevent code repetition ([d4a28cb](https://github.com/ing-bank/popmon/commits/d4a28cbfe6e3dcc8cf5ed1d92b1d679eb06aaab7)) +* convert dict constructor to dict literal ([331d8b1](https://github.com/ing-bank/popmon/commits/331d8b17620d90fef24360232fcdf2ab84e40b92)), ([b3d4d88](https://github.com/ing-bank/popmon/commits/b3d4d889cc4a38a4f2a0d29d87c5ec4ba417cfbf)), ([9e5188c](https://github.com/ing-bank/popmon/commits/9e5188c80dd709885311a2e041a06c86c6caa897)), ([c9518ab](https://github.com/ing-bank/popmon/commits/c9518abc1e52789193d928e336eddb38d2b5881e)) +* directly pass arguments ([bf33807](https://github.com/ing-bank/popmon/commits/bf338075f4dee3c8e00e8997497e32063482b8f3)) + + +### 🐛 Bug fixes + +* copyright in license ([b16fd99](https://github.com/ing-bank/popmon/commits/b16fd993a0c4bed2ee50991ac2863e2f196c65ae)), ([60a22dd](https://github.com/ing-bank/popmon/commits/60a22dd36fa0ec9c2a6a6dc97eccb72509841f16)) +* extend TL config ([eba10ed](https://github.com/ing-bank/popmon/commits/eba10edb5d784c6a7644cef999d42bad22a8d7a2)) +* mismatch between stats displayed in the traffic lights section and comparisons due to pattern matching bug. ([5ffb0a9](https://github.com/ing-bank/popmon/commits/5ffb0a97e032f5500f65d01a88a93f2b60e99471)) + + +### 📖 Documentation + +* add explanation for hiding green TLs by default ([81095af](https://github.com/ing-bank/popmon/commits/81095af73b5b0157c4d6d373142a49747deda22b)) +* codemotion article ([1e20f30](https://github.com/ing-bank/popmon/commits/1e20f304567afd1edac987521c8026e2146f5d17)) +* include changelog in docs ([a2237d0](https://github.com/ing-bank/popmon/commits/a2237d088439ecd563fc68ad82696855bf8f6ec4)) + + +### ⬆️ Dependencies + +* pre-commit updates ([6f55155](https://github.com/ing-bank/popmon/commits/6f55155a920de83e33c6129ddd85ecaf682eff47)), ([8fc5518](https://github.com/ing-bank/popmon/commits/8fc5518c5963f6a9dfdcf5e7f8025faa69b0f983)), ([e02adf1](https://github.com/ing-bank/popmon/commits/e02adf1dab9e30815fec4150eedbb01e15a81509)) +* **histogrammar:** popmon working with hgr v1.0.23 ([#101](https://github.com/ing-bank/popmon/issues/101)) ([d4a986e](https://github.com/ing-bank/popmon/commits/d4a986ed0add983721d2a60fbefc385fe2ed7ed3)) +* Pin ing-matplotlib-theme dependency to master (closes #131) ([1e20f30](https://github.com/ing-bank/popmon/commit/f1ed364045d4286ab3e144034fd8cf2c1e3aef89))