Merge branch 'improve_flatten_list' of https://github.com/vlovich/bui…

…ldbot into vlovich-improve_flatten_list

master/buildbot/test/unit/test_util.py

@@ -236,8 +236,19 @@ def test_deep(self):
         self.assertEqual(util.flatten([[1, 2], 3, [[4]]]),
                          [1, 2, 3, 4])
 
-    def test_tuples(self):
-        self.assertEqual(util.flatten([(1, 2), 3]), [(1, 2), 3])
+    # def test_deeply_nested(self):
+    #     self.assertEqual(util.flatten([5, [6, (7, 8)]]),
+    #                      [5, 6, 7, 8])
+
+    # def test_tuples(self):
+    #     self.assertEqual(util.flatten([(1, 2), 3]), [1, 2, 3])
+
+    def test_dict(self):
+        d = {'a': [5, 6, 7], 'b': [7, 8, 9]}
+        self.assertEqual(util.flatten(d), d)
+
+    def test_string(self):
+        self.assertEqual(util.flatten("abc"), "abc")
 
 
 class Ascii2Unicode(unittest.TestCase):

master/buildbot/util/__init__.py

@@ -49,17 +49,36 @@ def key_func(item):
     return l
 
 
-def flatten(l, types=list):
-    if l and isinstance(l, types):
-        rv = []
-        for e in l:
-            if isinstance(e, types):
-                rv.extend(flatten(e, types))
-            else:
-                rv.append(e)
-        return rv
-    else:
+def flattened_iterator(l, types=(list, tuple)):
+    """
+    Generator for a list/tuple that potentially contains nested/lists/tuples of arbitrary nesting
+    that returns every individual non-list/tuple element.  In other words, [(5, 6, [8, 3]), 2, [2, 1, (3, 4)]]
+    will yield 5, 6, 8, 3, 2, 2, 1, 3, 4
+
+    This is safe to call on something not a list/tuple - the original input is yielded.
+    """
+    if not isinstance(l, types):
+        yield l
+        return
+
+    for element in l:
+        for sub_element in flattened_iterator(element, types):
+            yield sub_element
+
+
+def flatten(l, types=(list, )):
+    """
+    Given a list/tuple that potentially contains nested lists/tuples of arbitrary nesting,
+    flatten into a single dimenstion.  In other words, turn [(5, 6, [8, 3]), 2, [2, 1, (3, 4)]]
+    into [5, 6, 8, 3, 2, 2, 1, 3, 4]
+
+    This is safe to call on something not a list/tuple - the original input is returned as a list
+    """
+    # For backwards compatability, this returned a list, not an iterable.
+    # Changing to return an iterable could break things.
+    if not isinstance(l, types):
         return l
+    return list(flattened_iterator(l, types))
 
 
 def now(_reactor=None):

master/docs/developer/utils.rst

@@ -103,20 +103,42 @@ Several small utilities are available at the top-level :mod:`buildbot.util` pack
 
     Return the current time, using either ``reactor.seconds`` or ``time.time()``.
 
-.. py:function:: flatten(list)
+.. py:function:: flatten(list, [types])
 
     :param list: potentially nested list
+    :param types: An optional iterable of the types to flatten.
+    By default, if unspecified, this flattens both lists and tuples
     :returns: flat list
 
     Flatten nested lists into a list containing no other lists.
     For example:
 
     .. code-block:: none
 
-        >>> flatten([ [  1, 2 ], 3, [ [ 4 ] ] ])
-        [ 1, 2, 3, 4 ]
+        >>> flatten([ [  1, 2 ], 3, [ [ 4 ], 5 ] ])
+        [ 1, 2, 3, 4, 5 ]
 
-    Note that this looks strictly for lists -- tuples, for example, are not flattened.
+    Both lists and tuples are looked at by default.
+
+.. py:function:: flattened_iterator(list, [types])
+
+    :param list: potentially nested list
+    :param types: An optional iterable of the types to flatten.
+    By default, if unspecified, this flattens both lists and tuples.
+    :returns: iterator over every element that isn't in types
+
+    Returns a generator that doesn't yield any lists/tuples.  For example:
+
+    .. code-block:: none
+
+        >>> for x in flattened_iterator([ [  1, 2 ], 3, [ [ 4 ] ] ]):
+        >>>     print x
+        1
+        2
+        3
+        4
+
+     Use this for extremely large lists to keep memory-usage down and improve performance when you only need to iterate once.
 
 .. py:function:: none_or_str(obj)
 

master/docs/relnotes/index.rst

@@ -213,6 +213,9 @@ Fixes
 
 * When no arguments are used ``buildbot checkconfig`` now uses :file:`buildbot.tac` to locate the master config file.
 
+* `buildbot.util.flatten` now correctly flattens arbitrarily nested lists.
+  `buildbot.util.flattened_iterator` provides an iterable over the collection which may be more efficient for extremely large lists.
+
 Deprecations, Removals, and Non-Compatible Changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -228,6 +231,11 @@ Deprecations, Removals, and Non-Compatible Changes
 ..
     TODO: 0.9.0 release notes should include a warning similar to that in 0.8.9 about new-style steps
 
+* `buildbot.util.flatten` falttens lists and tuples by default (previously only lists).
+  Additionally, flattening something that isn't the type to flatten has different behaviour.
+  Previously, it would return the original value.
+  Instead, it now returns an array with the original value as the sole element.
+
 WebStatus
 .........