fix flatten handling of nulls/nones (#70141)

* fix flatten loop control issue (break -> continue)

fix issue #69012

(cherry picked from commit 2127be5)

* fixed null break bug and added option to include

 fixes #69012
 fixes #69013

Co-authored-by: pseudocoder <borisovano@users.noreply.github.com>
Co-authored-by: David Shrewsbury <Shrews@users.noreply.github.com>

changelogs/fragments/fix_lp_flat.yml

@@ -0,0 +1,2 @@
+bugfixes:
+ - Continue execution when  'flatten' filter when it hits a None/null value as part of the list.

docs/docsite/rst/user_guide/playbooks_filters.rst

@@ -874,6 +874,14 @@ Flatten only the first level of a list (akin to the `items` lookup)::
     {{ [3, [4, [2]] ] | flatten(levels=1) }}
 
 
+.. versionadded:: 2.10
+
+Preserve nulls in a list, by default flatten removes them. ::
+
+    {{ [3, None, [4, [2]] ] | flatten(levels=1, skip_nulls=False) }}
+
+
+
 .. _set_theory_filters:
 
 Selecting from sets or lists (set theory)

lib/ansible/plugins/filter/core.py

@@ -467,19 +467,19 @@ def b64decode(string, encoding='utf-8'):
     return to_text(base64.b64decode(to_bytes(string, errors='surrogate_or_strict')), encoding=encoding)
 
 
-def flatten(mylist, levels=None):
+def flatten(mylist, levels=None, skip_nulls=True):
 
     ret = []
     for element in mylist:
-        if element in (None, 'None', 'null'):
-            # ignore undefined items
-            break
+        if skip_nulls and element in (None, 'None', 'null'):
+            # ignore null items
+            continue
         elif is_sequence(element):
             if levels is None:
-                ret.extend(flatten(element))
+                ret.extend(flatten(element, skip_nulls=skip_nulls))
             elif levels >= 1:
                 # decrement as we go down the stack
-                ret.extend(flatten(element, levels=(int(levels) - 1)))
+                ret.extend(flatten(element, levels=(int(levels) - 1), skip_nulls=skip_nulls))
             else:
                 ret.append(element)
         else:

test/integration/targets/filter_core/tasks/main.yml

@@ -123,13 +123,20 @@
       - "'unsupported hash type' in unsupported_hash_type_res.msg"
 
 - name: Flatten tests
+  tags: flatten
   block:
     - name: use flatten
       set_fact:
         flat_full: '{{orig_list|flatten}}'
         flat_one: '{{orig_list|flatten(levels=1)}}'
         flat_two: '{{orig_list|flatten(levels=2)}}'
         flat_tuples: '{{ [1,3] | zip([2,4]) | list | flatten }}'
+        flat_full_null: '{{list_with_nulls|flatten(skip_nulls=False)}}'
+        flat_one_null: '{{list_with_nulls|flatten(levels=1, skip_nulls=False)}}'
+        flat_two_null: '{{list_with_nulls|flatten(levels=2, skip_nulls=False)}}'
+        flat_full_nonull: '{{list_with_nulls|flatten(skip_nulls=True)}}'
+        flat_one_nonull: '{{list_with_nulls|flatten(levels=1, skip_nulls=True)}}'
+        flat_two_nonull: '{{list_with_nulls|flatten(levels=2, skip_nulls=True)}}'
 
     - name: Verify flatten filter works as expected
       assert:
@@ -138,8 +145,18 @@
           - flat_one == [1, 2, 3, [4, [5]], 6, 7]
           - flat_two == [1, 2, 3, 4, [5], 6, 7]
           - flat_tuples == [1, 2, 3, 4]
+          - flat_full_null == [1, 'None', 3, 4, 5, 6, 7]
+          - flat_one_null == [1, 'None', 3, [4, [5]], 6, 7]
+          - flat_two_null == [1, 'None', 3, 4, [5], 6, 7]
+          - flat_full_nonull == [1, 3, 4, 5, 6, 7]
+          - flat_one_nonull == [1, 3, [4, [5]], 6, 7]
+          - flat_two_nonull == [1, 3, 4, [5], 6, 7]
+          - list_with_subnulls|flatten(skip_nulls=False) ==  [1, 2, 'None', 4, 5, 6, 7]
+          - list_with_subnulls|flatten(skip_nulls=True) ==  [1, 2, 4, 5, 6, 7]
   vars:
     orig_list: [1, 2, [3, [4, [5]], 6], 7]
+    list_with_nulls: [1, None, [3, [4, [5]], 6], 7]
+    list_with_subnulls: [1, 2, [None, [4, [5]], 6], 7]
 
 - name: Test base64 filter
   assert: