Merge pull request #918 from untergeek/feature/733

Add the period filter

curator/cli.py

@@ -121,7 +121,9 @@ def cli(config, dry_run, action_file):
     #########################################
     ### Start working on the actions here ###
     #########################################
+    logger.debug('action_file: {0}'.format(action_file))
     action_config = get_yaml(action_file)
+    logger.debug('action_config: {0}'.format(action_config))
     action_dict = validate_actions(action_config)
     actions = action_dict['actions']
     logger.debug('Full list of actions: {0}'.format(actions))

curator/defaults/filter_elements.py

@@ -57,6 +57,12 @@ def max_num_segments(**kwargs):
         Required('max_num_segments'): All(Coerce(int), Range(min=1))
     }
 
+def range_from(**kwargs):
+    return { Required('range_from'): Coerce(int) }
+
+def range_to(**kwargs):
+    return { Required('range_to'): Coerce(int) }
+
 def reverse(**kwargs):
     # Only used with space filtertype
     # Should be ignored if `use_age` is True
@@ -98,11 +104,18 @@ def timestring(**kwargs):
 def unit(**kwargs):
     # This setting is only used with the age filtertype, or with the space
     # filtertype if use_age is set to True.
-    return {
-        Required('unit'): Any(
-            'seconds', 'minutes', 'hours', 'days', 'weeks', 'months', 'years'
-        )
-    }
+    if 'period' in kwargs and kwargs['period']:
+        return {
+            Required('unit'): Any(
+                'hours', 'days', 'weeks', 'months', 'years'
+            )
+        }    
+    else:
+        return {
+            Required('unit'): Any(
+                'seconds','minutes', 'hours', 'days', 'weeks', 'months', 'years'
+            )
+        }
 
 def unit_count(**kwargs):
     # This setting is only used with the age filtertype, or with the space
@@ -118,3 +131,10 @@ def value(**kwargs):
     # setting. There is a separate value option associated with the allocation
     # action, and the allocated filtertype.
     return { Required('value'): Any(str, unicode) }
+
+def week_starts_on(**kwargs):
+    return { 
+        Optional('week_starts_on', default='sunday'): Any(
+            'Sunday', 'sunday', 'SUNDAY', 'Monday', 'monday', 'MONDAY', None
+        )
+    }

curator/defaults/filtertypes.py

@@ -97,6 +97,18 @@ def pattern(action, config):
         filter_elements.exclude(),
     ]
 
+def period(action, config):
+    retval = [
+        filter_elements.unit(period=True),
+        filter_elements.range_from(),
+        filter_elements.range_to(),
+        filter_elements.week_starts_on(),
+        filter_elements.epoch(),
+        filter_elements.exclude(),
+    ]
+    retval += _age_elements(action, config)
+    return retval
+
 def space(action, config):
     retval = [
         filter_elements.disk_space(),

curator/defaults/settings.py

@@ -72,11 +72,12 @@ def index_filtertypes():
         'none',
         'opened',
         'pattern',
+        'period',
         'space',
     ]
 
 def snapshot_filtertypes():
-    return ['age', 'count', 'none', 'pattern', 'state']
+    return ['age', 'count', 'none', 'pattern', 'period', 'state']
 
 def all_filtertypes():
     return sorted(list(set(index_filtertypes() + snapshot_filtertypes())))
@@ -106,6 +107,8 @@ def structural_filter_elements():
         Optional('kind'): Any(str, unicode),
         Optional('max_num_segments'): Coerce(int),
         Optional('reverse'): Any(int, str, unicode, bool, None),
+        Optional('range_from'): Coerce(int),
+        Optional('range_to'): Coerce(int),
         Optional('source'): Any(str, unicode),
         Optional('state'): Any(str, unicode),
         Optional('stats_result'): Any(str, unicode, None),
@@ -114,4 +117,5 @@ def structural_filter_elements():
         Optional('unit_count'): Coerce(int),
         Optional('use_age'): Boolean(),
         Optional('value'): Any(int, float, str, unicode, bool),
+        Optional('week_starts_on'): Any(str, unicode, None),
     }

curator/indexlist.py

@@ -100,6 +100,7 @@ def __map_method(self, ft):
             'kibana': self.filter_kibana,
             'none': self.filter_none,
             'opened': self.filter_opened,
+            'period': self.filter_period,
             'pattern': self.filter_by_regex,
             'space': self.filter_by_space,
         }
@@ -432,21 +433,22 @@ def filter_by_age(self, source='name', direction=None, timestring=None,
         )
         for index in self.working_list():
             try:
+                age = int(self.index_info[index]['age'][self.age_keyfield])
                 msg = (
                     'Index "{0}" age ({1}), direction: "{2}", point of '
                     'reference, ({3})'.format(
                         index,
-                        int(self.index_info[index]['age'][self.age_keyfield]),
+                        age,
                         direction,
                         PoR
                     )
                 )
                 # Because time adds to epoch, smaller numbers are actually older
                 # timestamps.
                 if direction == 'older':
-                    agetest = self.index_info[index]['age'][self.age_keyfield] < PoR
+                    agetest = age < PoR
                 else:
-                    agetest = self.index_info[index]['age'][self.age_keyfield] > PoR
+                    agetest = age > PoR
                 self.__excludify(agetest, exclude, index, msg)
             except KeyError:
                 self.loggit.debug(
@@ -796,6 +798,73 @@ def filter_by_count(
             self.__excludify(condition, exclude, index, msg)
             idx += 1
 
+    def filter_period(
+        self, source='name', range_from=None, range_to=None, timestring=None,
+        unit=None, field=None, stats_result='min_value', 
+        week_starts_on='sunday', epoch=None, exclude=False,
+        ):
+        """
+        Match `indices` within ages within a given period.
+
+        :arg source: Source of index age. Can be one of 'name', 'creation_date',
+            or 'field_stats'
+        :arg range_from: How many ``unit``s in the past/future is the origin?
+        :arg range_to: How many ``unit``s in the past/future is the end point?
+        :arg timestring: An strftime string to match the datestamp in an index
+            name. Only used for index filtering by ``name``.
+        :arg unit: One of ``hours``, ``days``, ``weeks``, ``months``, or 
+            ``years``.
+        :arg unit_count: The number of ``unit``s. ``unit_count`` * ``unit`` will
+            be calculated out to the relative number of seconds.
+        :arg field: A timestamp field name.  Only used for ``field_stats`` based
+            calculations.
+        :arg stats_result: Either `min_value` or `max_value`.  Only used in
+            conjunction with `source`=``field_stats`` to choose whether to
+            reference the minimum or maximum result value.
+        :arg week_starts_on: Either ``sunday`` or ``monday``. Default is 
+            ``sunday``
+        :arg epoch: An epoch timestamp used to establish a point of reference 
+            for calculations. If not provided, the current time will be used.
+        :arg exclude: If `exclude` is `True`, this filter will remove matching
+            indices from `indices`. If `exclude` is `False`, then only matching
+            indices will be kept in `indices`.
+            Default is `False`
+        """
+
+        self.loggit.debug('Filtering indices by age')
+        try:
+            start, end = date_range(
+                unit, range_from, range_to, epoch, week_starts_on=week_starts_on
+            )
+        except Exception as e:
+            report_failure(e)
+
+        self._calculate_ages(
+            source=source, timestring=timestring, field=field,
+            stats_result=stats_result
+        )
+        for index in self.working_list():
+            try:
+                age = int(self.index_info[index]['age'][self.age_keyfield])
+                msg = (
+                    'Index "{0}" age ({1}), period start: "{2}", period '
+                    'end, "{3}"'.format(
+                        index,
+                        age,
+                        start,
+                        end
+                    )
+                )
+                # Because time adds to epoch, smaller numbers are actually older
+                # timestamps.
+                inrange = ((age >= start) and (age <= end))
+                self.__excludify(inrange, exclude, index, msg)
+            except KeyError:
+                self.loggit.debug(
+                    'Index "{0}" does not meet provided criteria. '
+                    'Removing from list.'.format(index, source))
+                self.indices.remove(index)
+
     def iterate_filters(self, filter_dict):
         """
         Iterate over the filters defined in `config` and execute them.

curator/snapshotlist.py

@@ -88,6 +88,7 @@ def __map_method(self, ft):
             'count': self.filter_by_count,
             'none': self.filter_none,
             'pattern': self.filter_by_regex,
+            'period': self.filter_period,
             'state': self.filter_by_state,
         }
         return methods[ft]
@@ -385,6 +386,59 @@ def filter_by_count(
             self.__excludify(condition, exclude, snap, msg)
             idx += 1
 
+    def filter_period(
+        self, source='name', range_from=None, range_to=None, timestring=None,
+        unit=None, field=None, stats_result='min_value', 
+        week_starts_on='sunday', epoch=None, exclude=False,
+        ):
+        """
+        Match `indices` within ages within a given period.
+
+        :arg source: Source of snapshot age. Can be 'name', or 'creation_date'.
+        :arg range_from: How many ``unit``s in the past/future is the origin?
+        :arg range_to: How many ``unit``s in the past/future is the end point?
+        :arg timestring: An strftime string to match the datestamp in an
+            snapshot name. Only used for snapshot filtering by ``name``.
+        :arg unit: One of ``hours``, ``days``, ``weeks``, ``months``, or 
+            ``years``.
+        :arg week_starts_on: Either ``sunday`` or ``monday``. Default is 
+            ``sunday``
+        :arg epoch: An epoch timestamp used to establish a point of reference 
+            for calculations. If not provided, the current time will be used.
+        :arg exclude: If `exclude` is `True`, this filter will remove matching
+            indices from `indices`. If `exclude` is `False`, then only matching
+            indices will be kept in `indices`.
+            Default is `False`
+        """
+
+        self.loggit.debug('Filtering snapshots by period')
+        try:
+            start, end = date_range(
+                unit, range_from, range_to, epoch, week_starts_on=week_starts_on
+            )
+        except Exception as e:
+            report_failure(e)
+        self._calculate_ages(source=source, timestring=timestring)
+        for snapshot in self.working_list():
+            if not self.snapshot_info[snapshot][self.age_keyfield]:
+                self.loggit.debug('Removing snapshot {0} for having no age')
+                self.snapshots.remove(snapshot)
+                continue
+            age = fix_epoch(self.snapshot_info[snapshot][self.age_keyfield])
+            msg = (
+                'Snapshot "{0}" age ({1}), period start: "{2}", period '
+                'end, ({3})'.format(
+                    snapshot,
+                    age,
+                    start,
+                    end
+                )
+            )
+            # Because time adds to epoch, smaller numbers are actually older
+            # timestamps.
+            inrange = ((age >= start) and (age <= end))
+            self.__excludify(inrange, exclude, snapshot, msg)
+
     def iterate_filters(self, config):
         """
         Iterate over the filters defined in `config` and execute them.