Merge pull request #298 from untergeek/enhancement/297

Addresses concerns raised in #297

Changelog.rst

@@ -6,6 +6,12 @@ Changelog
 3.0.1 (? ? ?)
 -------------
 
+**General**
+
+ * Renamed horribly named `regex_iterate` method to `apply_filter` #298 (untergeek)
+ * Added `build_filter` method to mimic CLI calls. #298 (untergeek)
+ * Added Examples page in the API documentation. #298 (untergeek)
+
 **Bug fixes**
 
  * Refactored to show `--dry-run` info for `--disk-space` calls. Reported in

curator/api/filter.py

@@ -5,6 +5,14 @@
 import logging
 logger = logging.getLogger(__name__)
 
+REGEX_MAP = {
+    'timestring': r'^.*{0}.*$',
+    'newer_than': r'(?P<date>{0})',
+    'older_than': r'(?P<date>{0})',
+    'prefix': r'^{0}.*$',
+    'suffix': r'^.*{0}$',
+}
+
 DATE_REGEX = {
     'Y' : '4',
     'y' : '2',
@@ -17,7 +25,66 @@
     'S' : '2',
 }
 
-def regex_iterate(
+def build_filter(
+        kindOf=None, time_unit=None, timestring=None,
+        groupname='date', value=None, exclude=False,
+    ):
+    """
+    Return a filter object based on the arguments.
+
+    :arg kindOf: Can be one of:
+        [older_than|newer_than|suffix|prefix|regex|timestring].
+        This option defines what kind of filter you will be building.
+    :arg exclude: If `True`, exclude matches rather than include
+    :arg groupname: The name of a named capture in pattern.  Currently only acts
+        on 'date'
+    :arg timestring: An strftime string to match the datestamp in an index name.
+        Only used for time-based filtering.
+    :arg time_unit: One of ``hours``, ``days``, ``weeks``, ``months``.
+        (default: ``days``). Only used for time-based filtering.
+    :arg value: Depends on `kindOf`. It's a time-unit multiplier for
+        `older_than` and `newer_than`.  It is the strftime string if `kindOf` is
+        `timestring`. It's used to build the regular expression for other kinds.
+    """
+    if kindOf not in [  'older_than', 'newer_than', 'regex',
+                    'exclude', 'prefix', 'suffix', 'timestring'   ]:
+        logger.error('{0}: Invalid value for kindOf'.format(kindOf))
+        return {}
+    # Stop here if None or empty value
+    if not value:
+        return {}
+    else:
+        argdict = {}
+
+    if kindOf in ['older_than', 'newer_than']:
+        if not time_unit:
+            logger.error("older_than and newer_than require time_unit parameter")
+            return {}
+        if not timestring:
+            logger.error("older_than and newer_than require timestring parameter")
+            return {}
+        argdict = {  "groupname":groupname, "time_unit":time_unit,
+                    "timestring": timestring, "value": value,
+                    "method": kindOf }
+        date_regex = get_date_regex(timestring)
+        regex = REGEX_MAP[kindOf].format(date_regex)
+    elif kindOf == 'timestring':
+        regex = r'^.*{0}.*$'.format(get_date_regex(value))
+    elif kindOf == 'regex':
+        regex = r'{0}'.format(value)
+    elif kindOf in ['prefix', 'suffix']:
+        regex = REGEX_MAP[kindOf].format(value)
+
+    if kindOf == 'exclude':
+        regex = r'{0}'.format(value)
+        argdict['exclude'] = True
+
+    logger.debug("REGEX = {0}".format(regex))
+    argdict['pattern'] = regex
+    logger.debug("Added filter: {0}".format(argdict))
+    return argdict
+
+def apply_filter(
     items, pattern=None, exclude=False, groupname=None, timestring=None,
     time_unit=None, method=None, value=None, utc_now=None):
     """Iterate over all items in the list and return a list of matches

curator/cli/index_selection.py

@@ -76,7 +76,7 @@ def indices(ctx, newer_than, older_than, prefix, suffix, time_unit,
         if all_indices and not 'exclude' in f:
             continue
         logger.debug('Filter: {0}'.format(f))
-        working_list = regex_iterate(working_list, **f)
+        working_list = apply_filter(working_list, **f)
 
     if ctx.parent.info_name == "delete": # Protect against accidental delete
         logger.info("Pruning Kibana-related indices to prevent accidental deletion.")

curator/cli/snapshot_selection.py

@@ -71,7 +71,7 @@ def snapshots(ctx, newer_than, older_than, prefix, suffix, time_unit,
         if all_snapshots and not 'exclude' in f:
             continue
         logger.debug('Filter: {0}'.format(f))
-        working_list = regex_iterate(working_list, **f)
+        working_list = apply_filter(working_list, **f)
 
     # If there are manually added snapshots, we will add them here
     working_list.extend(in_list(snapshot, snapshots))

docs/examples.rst

@@ -0,0 +1,242 @@
+.. _examples:
+
+Examples
+========
+
+`build_filter` Examples
+---------------------
+
+Filter indices by prefix
+++++++++++++++++++++++++
+
+This example will generate a list of indices matching the prefix, 'logstash'.
+The effective regular expression would be: `^logstash.*$`
+`filter` is a dictionary object.  We send it as key word arguments (kwargs) to
+`apply_filter`.  Filter's contents: ``{'pattern': '^logstash.*$'}``
+
+::
+
+    import elasticsearch
+    import curator
+
+    client = elasticsearch.Elasticsearch()
+
+    indices = curator.get_indices(client)
+    filter = curator.build_filter(kindOf='prefix', value='logstash')
+    working_list = curator.apply_filter(indices, **filter)
+
+The contents of `working_list` would then only be indices matching the `prefix`.
+
+
+Filter indices by suffix
+++++++++++++++++++++++++
+
+This example will generate a list of indices matching the suffix, '-prod'.
+The effective regular expression would be: `^.*-prod$`
+`filter` is a dictionary object.  We send it as key word arguments (kwargs) to
+`apply_filter`.  Filter's contents: ``{'pattern': '^.*-prod$'}``
+
+::
+
+    import elasticsearch
+    import curator
+
+    client = elasticsearch.Elasticsearch()
+
+    indices = curator.get_indices(client)
+    filter = curator.build_filter(kindOf='suffix', value='-prod')
+    working_list = curator.apply_filter(indices, **filter)
+
+The contents of `working_list` would then only be indices matching the `suffix`.
+
+
+Filter indices by time (older_than)
++++++++++++++++++++++++++++++++++++
+
+This example will generate a list of indices matching the following criteria:
+
+* Have a date string of ``%Y.%m.%d``
+* Use `days` as the unit of time measurement
+* Filter indices `older_than` 5 `days`
+
+`filter` is a dictionary object.  We send it as key word arguments (kwargs) to
+`apply_filter`.
+
+The resulting `filter` dictionary will be:
+
+::
+
+    {
+          'pattern': '(?P<date>\\d{4}\\.\\d{2}\\.\\d{2})', 'value': 5,
+          'groupname': 'date', 'time_unit': 'days',
+          'timestring': '%Y.%d.%m', 'method': 'older_than'
+    }
+
+::
+
+    import elasticsearch
+    import curator
+
+    client = elasticsearch.Elasticsearch()
+
+    indices = curator.get_indices(client)
+    filter = curator.build_filter(kindOf='suffix', value='-prod')
+    working_list = curator.apply_filter(indices, **filter)
+
+The contents of `working_list` would then only be indices matching these
+criteria.
+
+
+Filter indices by time (newer_than)
++++++++++++++++++++++++++++++++++++
+
+This example will generate a list of indices matching the following criteria:
+
+* Have a date string of ``%Y.%m.%d``
+* Use `days` as the unit of time measurement
+* Filter indices `newer_than` 5 `days`
+
+`filter` is a dictionary object.  We send it as key word arguments (kwargs) to
+`apply_filter`.
+
+The resulting `filter` dictionary will be:
+
+::
+
+    {
+          'pattern': '(?P<date>\\d{4}\\.\\d{2}\\.\\d{2})', 'value': 5,
+          'groupname': 'date', 'time_unit': 'days',
+          'timestring': '%Y.%d.%m', 'method': 'newer_than'
+    }
+
+::
+
+    import elasticsearch
+    import curator
+
+    client = elasticsearch.Elasticsearch()
+
+    indices = curator.get_indices(client)
+    filter = curator.build_filter(
+                kindOf='newer_than', value=5, time_unit='days',
+                timestring='%Y.%d.%m'
+             )
+    working_list = curator.apply_filter(indices, **filter)
+
+The contents of `working_list` would then only be indices matching these
+criteria.
+
+
+Filter indices by custom regular expression
++++++++++++++++++++++++++++++++++++++++++++
+
+This example will generate a list of indices matching a custom regular
+expression ``(your expression)``.
+
+``(your expression)`` needs to be a valid regular expression.
+
+`filter` is a dictionary object.  We send it as key word arguments (kwargs) to
+`apply_filter`.  Filter's contents: ``{'pattern': (your expression)}``
+
+::
+
+    import elasticsearch
+    import curator
+
+    client = elasticsearch.Elasticsearch()
+
+    indices = curator.get_indices(client)
+    filter = curator.build_filter(kindOf='regex', value=(your expression))
+    working_list = curator.apply_filter(indices, **filter)
+
+The contents of `working_list` would then only be indices matching
+(your expression)
+
+
+Filter indices by excluding matches
++++++++++++++++++++++++++++++++++++
+
+This example will generate a list of all indices `not` matching the pattern,
+'dev-'.
+
+The effective regular expression would be: `^dev-.*$`
+
+`filter` is a dictionary object.  We send it as key word arguments (kwargs) to
+`apply_filter`.  Filter's contents: ``{'pattern': 'dev-', 'exclude': True}``
+
+::
+
+    import elasticsearch
+    import curator
+
+    client = elasticsearch.Elasticsearch()
+
+    indices = curator.get_indices(client)
+    filter = curator.build_filter(kindOf='exclude', value='dev-')
+    working_list = curator.apply_filter(indices, **filter)
+
+The contents of `working_list` would then be all indices not matching the
+pattern, 'dev-'.
+
+.. note::
+
+    Any filter can become an `exclude` by adding ``'exclude':True`` to the
+    `filter` dictionary.
+
+Filter indices by time string as a pattern
+++++++++++++++++++++++++++++++++++++++++++
+
+This example will generate a list of indices having a matching time string,
+where `value` must be a valid python strftime string.
+
+`filter` is a dictionary object.  We send it as key word arguments (kwargs) to
+`apply_filter`.  Filter's contents:
+``{'pattern': '(?P<date>\\d{4}\\.\\d{2}\\.\\d{2})'}``
+
+::
+
+    import elasticsearch
+    import curator
+
+    client = elasticsearch.Elasticsearch()
+
+    indices = curator.get_indices(client)
+    filter = curator.build_filter(kindOf='timestring', value='%Y.%m.%d')
+    working_list = curator.apply_filter(indices, **filter)
+
+The contents of `working_list` would then only be indices having a matching
+time string.
+
+
+More complex example mimicking the CLI
+++++++++++++++++++++++++++++++++++++++
+
+This example will show time-series indices matching `prefix`, `older_than` 30
+`days` (the `time_unit`), and `newer_than` 60 `days`.
+
+::
+
+    import elasticsearch
+    import curator
+
+    client = elasticsearch.Elasticsearch()
+
+    indices = curator.get_indices(client)
+    filter_list = []
+    filter_list.append(curator.build_filter(kindOf='prefix', value='logstash'))
+    filter_list.append(
+        curator.build_filter(
+            kindOf='older_than', value=30, time_unit='days',
+            timestring='%Y.%d.%m'
+        )
+    )
+    filter_list.append(
+        curator.build_filter(
+            kindOf='newer_than', value=60, time_unit='days',
+            timestring='%Y.%d.%m'
+        )
+    )
+    working_list = indices
+    for filter in filter_list:
+        working_list = apply_filter(working_list, **filter)
+    curator.show(working_list)

docs/filters.rst

@@ -12,9 +12,13 @@ Sections
 Regex
 -----
 
-regex_iterate
+build_filter
+++++++++++++
+.. automethod:: curator.api.build_filter
+
+apply_filter
 +++++++++++++
-.. automethod:: curator.api.regex_iterate
+.. automethod:: curator.api.apply_filter
 
 get_date_regex
 ++++++++++++++

docs/index.rst

@@ -33,6 +33,9 @@ Example Usage
     curator.optimize_index(client, 'logstash-2014.08.31')
     curator.delete(client, ['logstash-2014.07.16', 'logstash-2014.07.17'])
 
+.. TIP::
+    See more examples in the :doc:`Examples </examples>` page.
+
 Features
 --------
 
@@ -42,6 +45,8 @@ The API methods fall into the following categories:
 * :doc:`Filters </filters>` are there to filter indices or snapshots based on provided criteria.
 * :doc:`Utilities </utilities>` are helper methods for commands and filters.
 
+Filtering indices is now handled by the :py:func:`curator.api.build_filter` method.
+
 Logging
 ~~~~~~~
 
@@ -64,6 +69,7 @@ Contents
    commands
    filters
    utilities
+   examples
    Changelog
 
 License