Merge pull request #1067 from dustin-decker/smaller-space

[FEATURE] Add support for returning indices smaller than `disk_space` in space filter

curator/defaults/filter_elements.py

@@ -110,6 +110,10 @@ def timestring(**kwargs):
     else:
         return { Optional('timestring', default=None): Any(str, unicode, None) }
 
+def threshold_behavior(**kwargs):
+    # This setting is only used with the space filtertype and defaults to 'greater_than'.
+    return { Optional('threshold_behavior', default='greater_than'): Any('greater_than', 'less_than') }
+
 def unit(**kwargs):
     # This setting is only used with the age filtertype, or with the space
     # filtertype if use_age is set to True.

curator/defaults/filtertypes.py

@@ -119,6 +119,7 @@ def space(action, config):
         filter_elements.reverse(),
         filter_elements.use_age(),
         filter_elements.exclude(),
+        filter_elements.threshold_behavior(),
     ]
     retval += _age_elements(action, config)
     return retval

curator/defaults/settings.py

@@ -119,6 +119,7 @@ def structural_filter_elements():
         Optional('state'): Any(str, unicode),
         Optional('stats_result'): Any(str, unicode, None),
         Optional('timestring'): Any(str, unicode, None),
+        Optional('threshold_behavior'): Any(str, unicode),
         Optional('unit'): Any(str, unicode),
         Optional('unit_count'): Coerce(int),
         Optional('unit_count_pattern'): Any(str, unicode),

curator/indexlist.py

@@ -488,7 +488,7 @@ def filter_by_age(self, source='name', direction=None, timestring=None,
     def filter_by_space(
         self, disk_space=None, reverse=True, use_age=False,
         source='creation_date', timestring=None, field=None,
-        stats_result='min_value', exclude=False):
+        stats_result='min_value', exclude=False, threshold_behavior='greater_than'):
         """
         Remove indices from the actionable list based on space
         consumed, sorted reverse-alphabetically by default.  If you set
@@ -507,7 +507,13 @@ def filter_by_space(
         ``max_value``, or ``min_value``.  The ``name`` `source` requires the
         timestring argument.
 
+        `threshold_behavior`, when set to `greater_than` (default), includes if it the index
+        tests to be larger than `disk_space`. When set to `less_than`, it includes if
+        the index is smaller than `disk_space`
+
         :arg disk_space: Filter indices over *n* gigabytes
+        :arg threshold_behavior: Size to filter, either ``greater_than`` or ``less_than``. Defaults
+            to ``greater_than`` to preserve backwards compatability.
         :arg reverse: The filtering direction. (default: `True`).  Ignored if
             `use_age` is `True`
         :arg use_age: Sort indices by age.  ``source`` is required in this
@@ -531,6 +537,12 @@ def filter_by_space(
         if not disk_space:
             raise MissingArgument('No value for "disk_space" provided')
 
+        if threshold_behavior not in ['greater_than', 'less_than']:
+            raise ValueError(
+                'Invalid value for "threshold_behavior": {0}'.format(
+                    threshold_behavior)
+            )
+
         disk_space = float(disk_space)
 
         disk_usage = 0.0
@@ -565,7 +577,12 @@ def filter_by_space(
                     index, byte_size(disk_usage), byte_size(disk_limit)
                 )
             )
-            self.__excludify((disk_usage > disk_limit), exclude, index, msg)
+            if threshold_behavior == 'greater_than':
+                self.__excludify((disk_usage > disk_limit),
+                                 exclude, index, msg)
+            elif threshold_behavior == 'less_than':
+                self.__excludify((disk_usage < disk_limit),
+                                 exclude, index, msg)
 
     def filter_kibana(self, exclude=True):
         """

docs/asciidoc/filter_elements.asciidoc

@@ -23,6 +23,7 @@
 * <<fe_state,state>>
 * <<fe_stats_result,stats_result>>
 * <<fe_timestring,timestring>>
+* <<fe_threshold_behavior,threshold_behavior>>
 * <<fe_unit,unit>>
 * <<fe_unit_count,unit_count>>
 * <<fe_unit_count_pattern,unit_count_pattern>>
@@ -612,6 +613,26 @@ include::inc_timestring_regex.asciidoc[]
 
 
 
+[[fe_threshold_behavior]]
+== threshold_behavior
+
+NOTE: This setting is only available in the <<filtertype_space,space>> filtertype.
+  This setting is optional, and defaults to `greater_than` to preserve backwards compatability.
+
+[source,yaml]
+-------------
+- filtertype: space
+  disk_space: 5
+  threshold_behavior: less_than
+-------------
+
+The value for this setting is `greater_than` (default) or `less_than`.
+
+When set to `less_than`, indices in less than `disk_space` gigabytes will be matched.
+When set to `greater_than` (default), indices larger than `disk_space` will be matched.
+
+
+
 [[fe_unit]]
 == unit
 

docs/asciidoc/filters.asciidoc

@@ -670,7 +670,7 @@ that whole unit will be selected, in this case, a month.
 == space
 
 This <<filtertype,filtertype>> will iterate over the actionable list and match
-indices when their cumulative disk consumption exceeds
+indices when their cumulative disk consumption is `greater_than` (default) or `less_than` than
 <<fe_disk_space,disk_space>> gigabytes.  They are first ordered by age,
 or by alphabet, so as to guarantee the oldest indices are deleted first. They
 will remain in, or be removed from the actionable list based on the value of
@@ -830,6 +830,7 @@ messages like these in the output:
 * <<fe_use_age,use_age>>
 * <<fe_source,source>> (required if `use_age` is `True`)
 * <<fe_timestring,timestring>> (required if `source` is `name`)
+* <<fe_threshold_behavior,fe_threshold_behavior>> (default is `greater_than`)
 * <<fe_field,field>> (required if `source` is `field_stats`)
 * <<fe_stats_result,stats_result>> (only used if `source` is `field_stats`)
 * <<fe_exclude,exclude>> (default is `False`)

test/unit/test_class_index_list.py

@@ -617,6 +617,25 @@ def test_filter_result_by_date_non_matching_timestring(self):
             source='name', timestring='%Y.%m.%d.%H'
         )
         self.assertEqual([], sorted(il.indices))
+    def test_filter_threshold_behavior(self):
+        client = Mock()
+        client.info.return_value = {'version': {'number': '5.0.0'} }
+        client.indices.get_settings.return_value = testvars.settings_two
+        client.cluster.state.return_value = testvars.clu_state_two
+        client.indices.stats.return_value = testvars.stats_two
+        client.field_stats.return_value = testvars.fieldstats_two
+        il = curator.IndexList(client)
+        il.filter_by_space(
+            disk_space=1.5, use_age=True,
+            threshold_behavior='less_than'
+        )
+        self.assertEqual(['index-2016.03.04'], sorted(il.indices))
+        il_b = curator.IndexList(client)
+        il_b.filter_by_space(
+            disk_space=1.5, use_age=True,
+            threshold_behavior='greater_than'
+        )
+        self.assertEqual(['index-2016.03.03'], sorted(il_b.indices))
     def test_filter_result_by_date_field_stats_raise(self):
         client = Mock()
         client.info.return_value = {'version': {'number': '5.0.0'} }