From 965c1f63816747d9dcd350b1451d8a2af6f45c0d Mon Sep 17 00:00:00 2001 From: pseudocoder Date: Fri, 17 Apr 2020 23:56:10 +0300 Subject: [PATCH 1/7] fix flatten loop control issue (break -> continue) fix issue #69012 (cherry picked from commit 2127be5ec5e6f892e1e303a3032f8aab394b97b2) --- lib/ansible/plugins/filter/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ansible/plugins/filter/core.py b/lib/ansible/plugins/filter/core.py index 9eecd79b79d2a0..38a317726954dd 100644 --- a/lib/ansible/plugins/filter/core.py +++ b/lib/ansible/plugins/filter/core.py @@ -473,7 +473,7 @@ def flatten(mylist, levels=None): for element in mylist: if element in (None, 'None', 'null'): # ignore undefined items - break + continue elif is_sequence(element): if levels is None: ret.extend(flatten(element)) From 5a12f1295a1794e696fa155e89ee4337bd9ecc85 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Thu, 18 Jun 2020 10:57:38 -0400 Subject: [PATCH 2/7] fixed null break bug and added option to include fixes #69012 fixes #69013 --- changelogs/fragments/fix_lp_flat.yml | 2 ++ lib/ansible/plugins/filter/core.py | 4 ++-- .../integration/targets/filter_core/tasks/main.yml | 14 ++++++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 changelogs/fragments/fix_lp_flat.yml diff --git a/changelogs/fragments/fix_lp_flat.yml b/changelogs/fragments/fix_lp_flat.yml new file mode 100644 index 00000000000000..84c5643c642ad0 --- /dev/null +++ b/changelogs/fragments/fix_lp_flat.yml @@ -0,0 +1,2 @@ +bugfixes: + - Continue execution when 'flatten' filter when it hits a None/null value as part of the list. diff --git a/lib/ansible/plugins/filter/core.py b/lib/ansible/plugins/filter/core.py index 38a317726954dd..4dd6b4f6f392ca 100644 --- a/lib/ansible/plugins/filter/core.py +++ b/lib/ansible/plugins/filter/core.py @@ -467,11 +467,11 @@ def b64decode(string, encoding='utf-8'): return to_text(base64.b64decode(to_bytes(string, errors='surrogate_or_strict')), encoding=encoding) -def flatten(mylist, levels=None): +def flatten(mylist, levels=None, skip_nulls=True): ret = [] for element in mylist: - if element in (None, 'None', 'null'): + if skip_nulls and element in (None, 'None', 'null'): # ignore undefined items continue elif is_sequence(element): diff --git a/test/integration/targets/filter_core/tasks/main.yml b/test/integration/targets/filter_core/tasks/main.yml index e8ef2345eb09ec..54056c24bb3481 100644 --- a/test/integration/targets/filter_core/tasks/main.yml +++ b/test/integration/targets/filter_core/tasks/main.yml @@ -113,6 +113,7 @@ - '"corned beef"|hash("haha, get it?") == None' - name: Flatten tests + tags: flatten block: - name: use flatten set_fact: @@ -120,6 +121,12 @@ flat_one: '{{orig_list|flatten(levels=1)}}' flat_two: '{{orig_list|flatten(levels=2)}}' flat_tuples: '{{ [1,3] | zip([2,4]) | list | flatten }}' + flat_full_null: '{{list_with_nulls|flatten(skip_nulls=False)}}' + flat_one_null: '{{list_with_nulls|flatten(levels=1, skip_nulls=False)}}' + flat_two_null: '{{list_with_nulls|flatten(levels=2, skip_nulls=False)}}' + flat_full_nonull: '{{list_with_nulls|flatten(skip_nulls=True)}}' + flat_one_nonull: '{{list_with_nulls|flatten(levels=1, skip_nulls=True)}}' + flat_two_nonull: '{{list_with_nulls|flatten(levels=2, skip_nulls=True)}}' - name: Verify flatten filter works as expected assert: @@ -128,8 +135,15 @@ - flat_one == [1, 2, 3, [4, [5]], 6, 7] - flat_two == [1, 2, 3, 4, [5], 6, 7] - flat_tuples == [1, 2, 3, 4] + - flat_full_null == [1, 'None', 3, 4, 5, 6, 7] + - flat_one_null == [1, 'None', 3, [4, [5]], 6, 7] + - flat_two_null == [1, 'None', 3, 4, [5], 6, 7] + - flat_full_nonull == [1, 3, 4, 5, 6, 7] + - flat_one_nonull == [1, 3, [4, [5]], 6, 7] + - flat_two_nonull == [1, 3, 4, [5], 6, 7] vars: orig_list: [1, 2, [3, [4, [5]], 6], 7] + list_with_nulls: [1, None, [3, [4, [5]], 6], 7] - name: Test base64 filter assert: From af555b2ee5c7e898e0a6b12cabb4c1e7dbf21373 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Fri, 19 Jun 2020 14:26:03 -0400 Subject: [PATCH 3/7] added docs for new option --- docs/docsite/rst/user_guide/playbooks_filters.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/docsite/rst/user_guide/playbooks_filters.rst b/docs/docsite/rst/user_guide/playbooks_filters.rst index 90b010b026e42d..1bd7a5016e3cb2 100644 --- a/docs/docsite/rst/user_guide/playbooks_filters.rst +++ b/docs/docsite/rst/user_guide/playbooks_filters.rst @@ -876,6 +876,14 @@ Flatten only the first level of a list (akin to the `items` lookup):: {{ [3, [4, [2]] ] | flatten(levels=1) }} +.. versionadded:: 2.10 + +Preserve nulls in a list, by default flatten removes them. :: + + {{ [3, None, [4, [2]] ] | flatten(levels=1, skip_nulls=False) }} + + + .. _set_theory_filters: Set theory filters From 86ee111ca02d47598c583f9514d81838a123578e Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Fri, 19 Jun 2020 14:27:23 -0400 Subject: [PATCH 4/7] c --- lib/ansible/plugins/filter/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ansible/plugins/filter/core.py b/lib/ansible/plugins/filter/core.py index 4dd6b4f6f392ca..0ecc8fa7a3da8d 100644 --- a/lib/ansible/plugins/filter/core.py +++ b/lib/ansible/plugins/filter/core.py @@ -472,7 +472,7 @@ def flatten(mylist, levels=None, skip_nulls=True): ret = [] for element in mylist: if skip_nulls and element in (None, 'None', 'null'): - # ignore undefined items + # ignore null items continue elif is_sequence(element): if levels is None: From f4740ce722ddffa3bf0b692c97d5e4e65e3ed066 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Tue, 23 Jun 2020 15:34:37 -0400 Subject: [PATCH 5/7] Update lib/ansible/plugins/filter/core.py Co-authored-by: David Shrewsbury --- lib/ansible/plugins/filter/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ansible/plugins/filter/core.py b/lib/ansible/plugins/filter/core.py index 0ecc8fa7a3da8d..bc98f00fa1bbf4 100644 --- a/lib/ansible/plugins/filter/core.py +++ b/lib/ansible/plugins/filter/core.py @@ -476,7 +476,7 @@ def flatten(mylist, levels=None, skip_nulls=True): continue elif is_sequence(element): if levels is None: - ret.extend(flatten(element)) + ret.extend(flatten(element, skip_nulls=skip_nulls)) elif levels >= 1: # decrement as we go down the stack ret.extend(flatten(element, levels=(int(levels) - 1))) From 1f742e819f56858c6dc9afa401e0bb630ea25222 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Tue, 23 Jun 2020 15:46:08 -0400 Subject: [PATCH 6/7] added new param for recurse --- lib/ansible/plugins/filter/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ansible/plugins/filter/core.py b/lib/ansible/plugins/filter/core.py index bc98f00fa1bbf4..899c21603317f0 100644 --- a/lib/ansible/plugins/filter/core.py +++ b/lib/ansible/plugins/filter/core.py @@ -479,7 +479,7 @@ def flatten(mylist, levels=None, skip_nulls=True): ret.extend(flatten(element, skip_nulls=skip_nulls)) elif levels >= 1: # decrement as we go down the stack - ret.extend(flatten(element, levels=(int(levels) - 1))) + ret.extend(flatten(element, levels=(int(levels) - 1), skip_nulls=skip_nulls)) else: ret.append(element) else: From 26a17a8987ab5f90859a6bdc41085285ccd9eebe Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Wed, 1 Jul 2020 16:49:50 -0400 Subject: [PATCH 7/7] added subnulls --- test/integration/targets/filter_core/tasks/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/integration/targets/filter_core/tasks/main.yml b/test/integration/targets/filter_core/tasks/main.yml index 54056c24bb3481..0fa745f63d4f86 100644 --- a/test/integration/targets/filter_core/tasks/main.yml +++ b/test/integration/targets/filter_core/tasks/main.yml @@ -141,9 +141,12 @@ - flat_full_nonull == [1, 3, 4, 5, 6, 7] - flat_one_nonull == [1, 3, [4, [5]], 6, 7] - flat_two_nonull == [1, 3, 4, [5], 6, 7] + - list_with_subnulls|flatten(skip_nulls=False) == [1, 2, 'None', 4, 5, 6, 7] + - list_with_subnulls|flatten(skip_nulls=True) == [1, 2, 4, 5, 6, 7] vars: orig_list: [1, 2, [3, [4, [5]], 6], 7] list_with_nulls: [1, None, [3, [4, [5]], 6], 7] + list_with_subnulls: [1, 2, [None, [4, [5]], 6], 7] - name: Test base64 filter assert: