Merge pull request #1060 from monkey3199/feature/shrink_copy_aliases

add feature "copy_aliases" for shrink action.

curator/actions.py

@@ -1749,6 +1749,7 @@ class Shrink(object):
     def __init__(self, ilo, shrink_node='DETERMINISTIC', node_filters={},
                 number_of_shards=1, number_of_replicas=1,
                 shrink_prefix='', shrink_suffix='-shrink',
+                copy_aliases=False,
                 delete_after=True, post_allocation={},
                 wait_for_active_shards=1,
                 extra_settings={}, wait_for_completion=True, wait_interval=9,
@@ -1766,6 +1767,9 @@ def __init__(self, ilo, shrink_node='DETERMINISTIC', node_filters={},
         :arg number_of_replicas: The number of replicas for the shrunk index
         :arg shrink_prefix: Prepend the shrunk index with this value
         :arg shrink_suffix: Append the value to the shrunk index (default: `-shrink`)
+        :arg copy_aliases: Whether to copy each source index aliases to target index after shrinking.
+            the aliases will be added to target index and deleted from source index at the same time(default: `False`)
+        :type copy_aliases: bool
         :arg delete_after: Whether to delete each index after shrinking. (default: `True`)
         :type delete_after: bool
         :arg post_allocation: If populated, the `allocation_type`, `key`, and 
@@ -1800,6 +1804,8 @@ def __init__(self, ilo, shrink_node='DETERMINISTIC', node_filters={},
         self.shrink_prefix    = shrink_prefix
         #: Instance variable. Internal reference to `shrink_suffix`
         self.shrink_suffix    = shrink_suffix
+        #: Instance variable. Internal reference to `copy_aliases`
+        self.copy_aliases = copy_aliases
         #: Instance variable. Internal reference to `delete_after`
         self.delete_after     = delete_after
         #: Instance variable. Internal reference to `post_allocation`
@@ -2027,6 +2033,19 @@ def pre_shrink_check(self, idx, dry_run=False):
         self._check_space(idx, dry_run)
         self.loggit.debug('FINISH PRE_SHRINK_CHECK')
 
+    def do_copy_aliases(self, source_idx, target_idx):
+        alias_actions = []
+        aliases = self.client.indices.get_alias(index=source_idx)
+        for alias in aliases[source_idx]['aliases']:
+            self.loggit.debug('alias: {0}'.format(alias))
+            alias_actions.append(
+                {'remove': {'index': source_idx, 'alias': alias}})
+            alias_actions.append(
+                {'add': {'index': target_idx, 'alias': alias}})
+        if alias_actions:
+            self.loggit.info('Copy alias actions: {0}'.format(alias_actions))
+            self.client.indices.update_aliases({ 'actions' : alias_actions })
+
     def do_dry_run(self):
         """
         Show what a regular run would do, but don't actually do it.
@@ -2043,6 +2062,9 @@ def do_dry_run(self):
                     self.loggit.info('DRY-RUN: Shrinking index "{0}" to "{1}" with settings: {2}, wait_for_active_shards={3}'.format(idx, target, self.body, self.wait_for_active_shards))
                     if self.post_allocation:
                         self.loggit.info('DRY-RUN: Applying post-shrink allocation rule "{0}" to index "{1}"'.format('index.routing.allocation.{0}.{1}:{2}'.format(self.post_allocation['allocation_type'], self.post_allocation['key'], self.post_allocation['value']), target))
+                    if self.copy_aliases:
+                        self.loggit.info('DRY-RUN: Copy source index aliases "{0}"'.format(self.client.indices.get_alias(idx)))
+                        #self.do_copy_aliases(idx, target)
                     if self.delete_after:
                         self.loggit.info('DRY-RUN: Deleting source index "{0}"'.format(idx))
         except Exception as e:
@@ -2090,6 +2112,10 @@ def do_action(self):
                     if self.post_allocation:
                         self.loggit.info('Applying post-shrink allocation rule "{0}" to index "{1}"'.format('index.routing.allocation.{0}.{1}:{2}'.format(self.post_allocation['allocation_type'], self.post_allocation['key'], self.post_allocation['value']), target))
                         self.route_index(target, self.post_allocation['allocation_type'], self.post_allocation['key'], self.post_allocation['value'])
+                    ## Copy aliases, if flagged
+                    if self.copy_aliases:
+                        self.loggit.info('Copy source index aliases "{0}"'.format(idx))
+                        self.do_copy_aliases(idx, target)
                     ## Delete, if flagged
                     if self.delete_after:
                         self.loggit.info('Deleting source index "{0}"'.format(idx))

curator/defaults/option_defaults.py

@@ -27,6 +27,9 @@ def delay():
             )
     }
 
+def copy_aliases():
+    return { Optional('copy_aliases', default=False): Any(bool, All(Any(str, unicode), Boolean())) }
+
 def delete_after():
     return { Optional('delete_after', default=True): Any(bool, All(Any(str, unicode), Boolean())) }
 

curator/validators/options.py

@@ -115,6 +115,7 @@ def action_specific(action):
             option_defaults.number_of_replicas(),
             option_defaults.shrink_prefix(),
             option_defaults.shrink_suffix(),
+            option_defaults.copy_aliases(),
             option_defaults.delete_after(),
             option_defaults.post_allocation(),
             option_defaults.wait_for_active_shards(action),

docs/Changelog.rst

@@ -25,7 +25,11 @@ Changelog
     allows you to specify hard dates for ``date_from`` and ``date_to``, while
     ``date_from_format`` and ``date_to_format`` are strftime strings to
     interpret the from and to dates. Requested in #1047 (untergeek)
-
+  * Add ``copy_aliases`` option to the ``shrink`` action. So this option is
+    only set in the ``shrink`` action. The default value of the option is
+    ``copy_aliases: 'False'`` and it does nothing. If you set to
+    ``copy_aliases: 'True'``, you could copy the aliases from the source index
+    to the target index. Requested in #1060 (monkey3199)
 **Bug Fixes**
 
   * Delete the target index (if it exists) in the event that a shrink fails.

examples/actions/shrink.yml

@@ -0,0 +1,48 @@
+---
+# Remember, leave a key empty if there is no value.  None will be a string,
+# not a Python "NoneType"
+#
+# Also remember that all examples have 'disable_action' set to True.  If you
+# want to use this action as a template, be sure to set this to False after
+# copying it.
+actions:
+  1:
+    action: shrink
+    description: >-
+      Shrink logstash indices older than 2 days on the node with the most
+      available space, excluding the node named 'not_this_node'.
+      Delete each source index after successful shrink, then reroute the shrunk
+      index with the provided parameters.
+    options:
+      disable_action: False
+      ignore_empty_list: True
+      shrink_node: DETERMINISTIC
+      node_filters:
+        permit_masters: False
+        exclude_nodes: ['not_this_node']
+      number_of_shards: 1
+      number_of_replicas: 1
+      shrink_prefix:
+      shrink_suffix: '-shrink'
+      copy_aliases: True
+      delete_after: True
+      post_allocation:
+        allocation_type: include
+        key: node_tag
+        value: cold
+      wait_for_active_shards: 1
+      extra_settings:
+        settings:
+          index.codec: best_compression
+      wait_for_completion: True
+      wait_interval: 9
+      max_wait: -1
+    filters:
+    - filtertype: pattern
+      kind: prefix
+      value: test_shrink-
+    - filtertype: age
+      source: creation_date
+      direction: older
+      unit: days
+      unit_count: 2

test/integration/test_shrink.py

@@ -79,13 +79,39 @@
 '        kind: prefix\n'
 '        value: my\n')
 
+copy_aliases = ('---\n'
+'actions:\n'
+'  1:\n'
+'    description: "Act on indices as filtered"\n'
+'    action: shrink\n'
+'    options:\n'
+'      shrink_node: {0}\n'
+'      node_filters:\n'
+'          {1}: {2}\n'
+'      number_of_shards: {3}\n'
+'      number_of_replicas: {4}\n'
+'      shrink_prefix: {5}\n'
+'      shrink_suffix: {6}\n'
+'      copy_aliases: {7}\n'
+'      delete_after: {8}\n'
+'    filters:\n'
+'      - filtertype: pattern\n'
+'        kind: prefix\n'
+'        value: my\n')
+
 class TestCLIShrink(CuratorTestCase):
     def builder(self, action_args):
         self.idx = 'my_index'
         suffix = '-shrunken'
         self.target = '{0}{1}'.format(self.idx, suffix)
         self.create_index(self.idx, shards=2)
         self.add_docs(self.idx)
+        # add alias in the source index
+        self.alias = 'my_alias'
+        alias_actions = []
+        alias_actions.append(
+            {'add': {'index': self.idx, 'alias': self.alias}})
+        self.client.indices.update_aliases({'actions': alias_actions})
         self.write_config(self.args['configfile'], testvars.client_config.format(host, port))
         self.write_config(self.args['actionfile'], action_args)
         test = clicktest.CliRunner()
@@ -180,3 +206,22 @@ def test_shrink_with_extras(self):
         self.assertTrue(settings[self.idx]['settings']['index']['routing']['allocation']['require']['_name'] == '')
         self.assertEqual(settings[self.target]['settings']['index']['codec'], 'best_compression')
         self.assertTrue(self.client.indices.exists_alias(index=self.target, name='my_alias'))
+    def test_shrink_with_copy_alias(self):
+        suffix = '-shrunken'
+        self.builder(
+            copy_aliases.format(
+                'DETERMINISTIC',
+                'permit_masters',
+                'True',
+                1,
+                0,
+                '',
+                suffix,
+                'True',
+                'True'
+            )
+        )
+        indices = curator.get_indices(self.client)
+        self.assertEqual(1, len(indices))  # Should have `my_index-shrunken`
+        self.assertEqual(indices[0], self.target)
+        self.assertTrue(self.client.indices.exists_alias(index=self.target, name=self.alias))