Merge pull request #1085 from untergeek/feature/1047

Add new absolute date feature to period filter

curator/defaults/filter_elements.py

@@ -15,6 +15,22 @@ def count(**kwargs):
     # This setting is only used with the count filtertype and is required
     return { Required('count'): All(Coerce(int), Range(min=1)) }
 
+def date_from(**kwargs):
+    # This setting is only used with the period filtertype.
+    return { Optional('date_from'): Any(str, unicode) }
+
+def date_from_format(**kwargs):
+    # This setting is only used with the period filtertype.
+    return { Optional('date_from_format'): Any(str, unicode) }
+
+def date_to(**kwargs):
+    # This setting is only used with the period filtertype.
+    return { Optional('date_to'): Any(str, unicode) }
+
+def date_to_format(**kwargs):
+    # This setting is only used with the period filtertype.
+    return { Optional('date_to_format'): Any(str, unicode) }
+
 def direction(**kwargs):
     # This setting is only used with the age filtertype.
     return { Required('direction'): Any('older', 'younger') }
@@ -66,11 +82,15 @@ def pattern(**kwargs):
         Optional('pattern'): Any(str, unicode)
     }
 
+def period_type(**kwargs):
+    # This setting is only used with the period filtertype.
+    return { Optional('period_type', default='relative'): Any('relative', 'absolute') }
+
 def range_from(**kwargs):
-    return { Required('range_from'): Coerce(int) }
+    return { Optional('range_from'): Coerce(int) }
 
 def range_to(**kwargs):
-    return { Required('range_to'): Coerce(int) }
+    return { Optional('range_to'): Coerce(int) }
 
 def reverse(**kwargs):
     # Only used with space filtertype
@@ -117,18 +137,11 @@ def threshold_behavior(**kwargs):
 def unit(**kwargs):
     # This setting is only used with the age filtertype, or with the space
     # filtertype if use_age is set to True.
-    if 'period' in kwargs and kwargs['period']:
-        return {
-            Required('unit'): Any(
-                'hours', 'days', 'weeks', 'months', 'years'
-            )
-        }    
-    else:
-        return {
-            Required('unit'): Any(
-                'seconds','minutes', 'hours', 'days', 'weeks', 'months', 'years'
-            )
-        }
+    return {
+        Required('unit'): Any(
+            'seconds', 'minutes', 'hours', 'days', 'weeks', 'months', 'years'
+        )
+    }
 
 def unit_count(**kwargs):
     # This setting is only used with the age filtertype, or with the space

curator/defaults/filtertypes.py

@@ -109,6 +109,11 @@ def period(action, config):
         filter_elements.epoch(),
         filter_elements.exclude(),
         filter_elements.intersect(),
+        filter_elements.period_type(),
+        filter_elements.date_from(),
+        filter_elements.date_from_format(),
+        filter_elements.date_to(),
+        filter_elements.date_to_format(),
     ]
     retval += _age_elements(action, config)
     return retval

curator/defaults/settings.py

@@ -102,6 +102,10 @@ def structural_filter_elements():
         Optional('aliases'): Any(str, [str], unicode, [unicode]),
         Optional('allocation_type'): Any(str, unicode),
         Optional('count'): Coerce(int),
+        Optional('date_from'): Any(str, unicode, None),
+        Optional('date_from_format'): Any(str, unicode, None),
+        Optional('date_to'): Any(str, unicode, None),
+        Optional('date_to_format'): Any(str, unicode, None),
         Optional('direction'): Any(str, unicode),
         Optional('disk_space'): float,
         Optional('epoch'): Any(Coerce(int), None),
@@ -112,6 +116,7 @@ def structural_filter_elements():
         Optional('kind'): Any(str, unicode),
         Optional('max_num_segments'): Coerce(int),
         Optional('pattern'): Any(str, unicode),
+        Optional('period_type'): Any(str, unicode),
         Optional('reverse'): Any(int, str, unicode, bool, None),
         Optional('range_from'): Coerce(int),
         Optional('range_to'): Coerce(int),

curator/indexlist.py

@@ -907,17 +907,30 @@ def filter_by_count(
                 idx += 1
 
     def filter_period(
-        self, source='name', range_from=None, range_to=None, timestring=None,
-        unit=None, field=None, stats_result='min_value', intersect=False,
-        week_starts_on='sunday', epoch=None, exclude=False,
+        self, period_type='relative', source='name', range_from=None, range_to=None,
+        date_from=None, date_to=None, date_from_format=None, date_to_format=None,
+        timestring=None, unit=None, field=None, stats_result='min_value',
+        intersect=False, week_starts_on='sunday', epoch=None, exclude=False,
         ):
         """
         Match `indices` within ages within a given period.
 
+        :arg period_type: Can be either ``absolute`` or ``relative``.  Default is
+            ``relative``.  ``date_from`` and ``date_to`` are required when using
+            ``period_type='absolute'`. ``range_from`` and ``range_to`` are
+            required with ``period_type='relative'`.
         :arg source: Source of index age. Can be one of 'name', 'creation_date',
             or 'field_stats'
         :arg range_from: How many ``unit`` (s) in the past/future is the origin?
         :arg range_to: How many ``unit`` (s) in the past/future is the end point?
+        :arg date_from: The simplified date for the start of the range
+        :arg date_to: The simplified date for the end of the range.  If this value
+            is the same as ``date_from``, the full value of ``unit`` will be
+            extrapolated for the range.  For example, if ``unit`` is ``months``,
+            and ``date_from`` and ``date_to`` are both ``2017.01``, then the entire
+            month of January 2017 will be the absolute date range.
+        :arg date_from_format: The strftime string used to parse ``date_from``
+        :arg date_to_format: The strftime string used to parse ``date_to``
         :arg timestring: An strftime string to match the datestamp in an index
             name. Only used for index filtering by ``name``.
         :arg unit: One of ``hours``, ``days``, ``weeks``, ``months``, or 
@@ -944,10 +957,30 @@ def filter_period(
         """
 
         self.loggit.debug('Filtering indices by age')
-        try:
-            start, end = date_range(
-                unit, range_from, range_to, epoch, week_starts_on=week_starts_on
+        if period_type not in ['absolute', 'relative']:
+            raise ValueError(
+                'Unacceptable value: {0} -- "period_type" must be either "absolute" or '
+                '"relative".'.format(period_type)
             )
+        if period_type == 'relative':
+            func = date_range
+            args = [unit, range_from, range_to, epoch]
+            kwgs = { 'week_starts_on': week_starts_on }
+            if type(range_from) != type(int()) or type(range_to) != type(int()):
+                raise ConfigurationError(
+                    '"range_from" and "range_to" must be integer values')
+        else:
+            func = absolute_date_range
+            args = [unit, date_from, date_to]
+            kwgs = { 'date_from_format': date_from_format, 'date_to_format': date_to_format }
+            for reqd in [date_from, date_to, date_from_format, date_to_format]:
+                if not reqd:
+                    raise ConfigurationError(
+                        'Must provide "date_from", "date_to", "date_from_format", and '
+                        '"date_to_format" with absolute period_type'
+                    )
+        try:
+            start, end = func(*args, **kwgs)
         except Exception as e:
             report_failure(e)
 

curator/utils.py

@@ -465,6 +465,77 @@ def date_range(unit, range_from, range_to, epoch=None, week_starts_on='sunday'):
         datetime.utcfromtimestamp(end_epoch).isoformat()))
     return (start_epoch, end_epoch)
 
+def absolute_date_range(
+        unit, date_from, date_to, 
+        date_from_format=None, date_to_format=None
+    ):
+    """
+    Get the epoch start time and end time of a range of ``unit``s, reckoning the
+    start of the week (if that's the selected unit) based on ``week_starts_on``,
+    which can be either ``sunday`` or ``monday``.
+
+    :arg unit: One of ``hours``, ``days``, ``weeks``, ``months``, or ``years``.
+    :arg date_from: The simplified date for the start of the range
+    :arg date_to: The simplified date for the end of the range.  If this value
+        is the same as ``date_from``, the full value of ``unit`` will be
+        extrapolated for the range.  For example, if ``unit`` is ``months``,
+        and ``date_from`` and ``date_to`` are both ``2017.01``, then the entire
+        month of January 2017 will be the absolute date range.
+    :arg date_from_format: The strftime string used to parse ``date_from``
+    :arg date_to_format: The strftime string used to parse ``date_to``
+    :rtype: tuple
+    """
+    acceptable_units = ['seconds', 'minutes', 'hours', 'days', 'weeks', 'months', 'years']
+    if unit not in acceptable_units:
+        raise ConfigurationError(
+            '"unit" must be one of: {0}'.format(acceptable_units))
+    if not date_from_format or not date_to_format:
+        raise ConfigurationError('Must provide "date_from_format" and "date_to_format"')
+    try:
+        start_epoch = datetime_to_epoch(get_datetime(date_from, date_from_format))
+        logger.debug('Start ISO8601 = {0}'.format(datetime.utcfromtimestamp(start_epoch).isoformat()))
+    except Exception as e:
+        raise ConfigurationError(
+            'Unable to parse "date_from" {0} and "date_from_format" {1}. '
+            'Error: {2}'.format(date_from, date_from_format, e)
+        )
+    try:
+        end_date = get_datetime(date_to, date_to_format)
+    except Exception as e:
+        raise ConfigurationError(
+            'Unable to parse "date_to" {0} and "date_to_format" {1}. '
+            'Error: {2}'.format(date_to, date_to_format, e)
+        )
+    # We have to iterate to one more month, and then subtract a second to get
+    # the last day of the correct month
+    if unit == 'months':
+        month = end_date.month
+        year = end_date.year
+        if month == 12:
+            year += 1
+            month = 1
+        else:
+            month += 1
+        new_end_date = datetime(year, month, 1, 0, 0, 0)
+        end_epoch = datetime_to_epoch(new_end_date) - 1
+    # Similarly, with years, we need to get the last moment of the year
+    elif unit == 'years':
+        new_end_date = datetime(end_date.year + 1, 1, 1, 0, 0, 0)
+        end_epoch = datetime_to_epoch(new_end_date) - 1
+    # It's not months or years, which have inconsistent reckoning...
+    else:
+        # This lets us use an existing method to simply add 1 more unit's worth
+        # of seconds to get hours, days, or weeks, as they don't change
+        # We use -1 as point of reference normally subtracts from the epoch
+        # and we need to add to it, so we'll make it subtract a negative value.
+        # Then, as before, subtract 1 to get the end of the period
+        end_epoch = get_point_of_reference(
+            unit, -1, epoch=datetime_to_epoch(end_date)) -1
+
+    logger.debug('End ISO8601 = {0}'.format(
+        datetime.utcfromtimestamp(end_epoch).isoformat()))
+    return (start_epoch, end_epoch)
+
 def byte_size(num, suffix='B'):
     """
     Return a formatted string indicating the size in bytes, with the proper

docs/Changelog.rst

@@ -18,6 +18,13 @@ Changelog
     when working with rollover indices.  Requested in #1044 (untergeek)
   * The ``es_repo_mgr create`` command now can take ``skip_repo_fs_check`` as
     an argument (default is False) #1072 (alexef)
+  * Add ``pattern_type`` feature expansion to the ``period`` filter.  The
+    default behavior is ``pattern_type='relative'``, which preserves existing
+    behaviors so users with existing configurations can continue to use them
+    without interruption.  The new ``pattern_type`` is ``absolute``, which
+    allows you to specify hard dates for ``date_from`` and ``date_to``, while
+    ``date_from_format`` and ``date_to_format`` are strftime strings to
+    interpret the from and to dates. Requested in #1047 (untergeek)
 
 **Bug Fixes**
 

docs/asciidoc/examples.asciidoc

@@ -53,6 +53,7 @@ actions:
         value: logstash-
         exclude:
       - filtertype: period
+        period_type: relative
         source: name
         range_from: -1
         range_to: -1
@@ -65,6 +66,7 @@ actions:
         kind: prefix
         value: logstash-
       - filtertype: period
+        period_type: relative
         source: name
         range_from: -2
         range_to: -2