Merge branch 'master' into 8.x

curator/_version.py

@@ -1,2 +1,2 @@
 """Curator Version"""
-__version__ = '8.0.10'
+__version__ = '8.0.11'

curator/actions/cold2frozen.py

@@ -127,56 +127,85 @@ def do_dry_run(self):
             )
             self.loggit.info(msg)
 
-
-    def do_action(self):
+    def mount_index(self, newidx, kwargs):
         """
         Call :py:meth:`~.elasticsearch.client.SearchableSnapshotsClient.mount` to mount the indices
         in :py:attr:`ilo` in the Frozen tier.
+        """
+        try:
+            self.loggit.debug('Mounting new index %s in frozen tier...', newidx)
+            self.client.searchable_snapshots.mount(**kwargs)
+        # pylint: disable=broad-except
+        except Exception as err:
+            report_failure(err)
 
-        Verify index looks good
-
+    def verify_mount(self, newidx):
+        """
+        Verify that newidx is a mounted index
+        """
+        self.loggit.debug('Verifying new index %s is mounted properly...', newidx)
+        idx_settings = self.client.indices.get(index=newidx)[newidx]
+        if is_idx_partial(idx_settings['settings']['index']):
+            self.loggit.info('Index %s is mounted for frozen tier', newidx)
+        else:
+            report_failure(SearchableSnapshotException(
+                f'Index {newidx} not a mounted searchable snapshot'))
+
+    def update_aliases(self, current_idx, newidx, aliases):
+        """
         Call :py:meth:`~.elasticsearch.client.IndicesClient.update_aliases` to update each new
         frozen index with the aliases from the old cold-tier index.
 
         Verify aliases look good.
-
+        """
+        alias_names = aliases.keys()
+        if not alias_names:
+            self.loggit.warning('No aliases associated with index %s', current_idx)
+        else:
+            self.loggit.debug('Transferring aliases to new index %s', newidx)
+            self.client.indices.update_aliases(
+                actions=get_alias_actions(current_idx, newidx, aliases))
+            verify = self.client.indices.get(index=newidx)[newidx]['aliases'].keys()
+            if alias_names != verify:
+                self.loggit.error(
+                    'Alias names do not match! %s does not match: %s', alias_names, verify)
+                report_failure(FailedExecution('Aliases failed to transfer to new index'))
+
+    def cleanup(self, current_idx, newidx):
+        """
         Call :py:meth:`~.elasticsearch.client.IndicesClient.delete` to delete the cold tier index.
         """
+        self.loggit.debug('Deleting old index: %s', current_idx)
         try:
-            for kwargs in self.action_generator():
-                aliases = kwargs.pop('aliases')
-                current_idx = kwargs.pop('current_idx')
-                newidx = kwargs['renamed_index']
-                # Actually do the mount
-                self.loggit.debug('Mounting new index %s in frozen tier...', newidx)
-                self.client.searchable_snapshots.mount(**kwargs)
-                # Verify it's mounted as a partial now:
-                self.loggit.debug('Verifying new index %s is mounted properly...', newidx)
-                idx_settings = self.client.indices.get(index=newidx)[newidx]
-                if is_idx_partial(idx_settings['settings']['index']):
-                    self.loggit.info('Index %s is mounted for frozen tier', newidx)
-                else:
-                    raise SearchableSnapshotException(
-                        f'Index {newidx} not a mounted searchable snapshot')
-                # Update Aliases
-                alias_names = aliases.keys()
-                if not alias_names:
-                    self.loggit.warning('No aliases associated with index %s', current_idx)
-                else:
-                    self.loggit.debug('Transferring aliases to new index %s', newidx)
-                    self.client.indices.update_aliases(
-                        actions=get_alias_actions(current_idx, newidx, aliases))
-                    verify = self.client.indices.get(index=newidx)[newidx]['aliases'].keys()
-                    if alias_names != verify:
-                        self.loggit.error(
-                            'Alias names do not match! %s does not match: %s', alias_names, verify)
-                        raise FailedExecution('Aliases failed to transfer to new index')
-                # Clean up old index
-                self.loggit.debug('Deleting old index: %s', current_idx)
-                self.client.indices.delete(index=current_idx)
-                self.loggit.info(
-                    'Successfully migrated %s to the frozen tier as %s', current_idx, newidx)
-
+            self.client.indices.delete(index=current_idx)
         # pylint: disable=broad-except
         except Exception as err:
             report_failure(err)
+        self.loggit.info(
+            'Successfully migrated %s to the frozen tier as %s', current_idx, newidx)
+
+    def do_action(self):
+        """
+        Do the actions outlined:
+
+        Mount
+        Verify
+        Update Aliases
+        Cleanup
+        """
+        for kwargs in self.action_generator():
+            aliases = kwargs.pop('aliases')
+            current_idx = kwargs.pop('current_idx')
+            newidx = kwargs['renamed_index']
+
+            # Mount the index
+            self.mount_index(newidx, kwargs)
+
+            # Verify it's mounted as a partial now:
+            self.verify_mount(newidx)
+
+            # Update Aliases
+            self.update_aliases(current_idx, newidx, aliases)
+
+            # Clean up old index
+            self.cleanup(current_idx, newidx)

curator/actions/create_index.py

@@ -74,10 +74,14 @@ def do_action(self):
         # Most likely error is a 400, `resource_already_exists_exception`
         except RequestError as err:
             match_list = ["index_already_exists_exception", "resource_already_exists_exception"]
-            if err.error in match_list and self.ignore_existing:
-                self.loggit.warning('Index %s already exists.', self.name)
+            if err.error in match_list:
+                if self.ignore_existing:
+                    self.loggit.warning('Index %s already exists.', self.name)
+                else:
+                    raise FailedExecution(f'Index {self.name} already exists.') from err
             else:
-                raise FailedExecution(f'Index {self.name} already exists.') from err
+                msg = f'Unable to create index "{self.name}". Error: {err.error}'
+                raise FailedExecution(msg) from err
         # pylint: disable=broad-except
         except Exception as err:
             report_failure(err)

curator/cli.py

@@ -2,8 +2,9 @@
 import sys
 import logging
 import click
-from es_client.defaults import LOGGING_SETTINGS
-from es_client.helpers.config import cli_opts, context_settings, get_args, get_client, get_config
+from es_client.defaults import OPTION_DEFAULTS
+from es_client.helpers.config import (
+    cli_opts, context_settings, generate_configdict, get_client, get_config, options_from_dict)
 from es_client.helpers.logging import configure_logging
 from es_client.helpers.utils import option_wrapper, prune_nones
 from curator.exceptions import ClientException
@@ -129,24 +130,17 @@ def process_action(client, action_def, dry_run=False):
         logger.debug('Doing the action here.')
         action_def.action_cls.do_action()
 
-def run(client_args, other_args, action_file, dry_run=False):
+def run(ctx: click.Context) -> None:
     """
-    Called by :py:func:`cli` to execute what was collected at the command-line
+    :param ctx: The Click command context 
 
-    :param client_args: The ClientArgs arguments object
-    :param other_args: The OtherArgs arguments object
-    :param action_file: The action configuration file
-    :param dry_run: Do not perform any changes
+    :type ctx: :py:class:`Context <click.Context>`
 
-    :type client_args: :py:class:`~.es_client.ClientArgs`
-    :type other_args: :py:class:`~.es_client.OtherArgs`
-    :type action_file: str
-    :type dry_run: bool
+    Called by :py:func:`cli` to execute what was collected at the command-line
     """
     logger = logging.getLogger(__name__)
-
-    logger.debug('action_file: %s', action_file)
-    all_actions = ActionsFile(action_file)
+    logger.debug('action_file: %s', ctx.params['action_file'])
+    all_actions = ActionsFile(ctx.params['action_file'])
     for idx in sorted(list(all_actions.actions.keys())):
         action_def = all_actions.actions[idx]
         ### Skip to next action if 'disabled'
@@ -160,16 +154,16 @@ def run(client_args, other_args, action_file, dry_run=False):
 
         # Override the timeout, if specified, otherwise use the default.
         if action_def.timeout_override:
-            client_args.request_timeout = action_def.timeout_override
+            ctx.obj['client_args'].request_timeout = action_def.timeout_override
 
         # Create a client object for each action...
         logger.info('Creating client object and testing connection')
 
         try:
             client = get_client(configdict={
                 'elasticsearch': {
-                    'client': prune_nones(client_args.asdict()),
-                    'other_settings': prune_nones(other_args.asdict())
+                    'client': prune_nones(ctx.obj['client_args'].asdict()),
+                    'other_settings': prune_nones(ctx.obj['other_args'].asdict())
                 }
             })
         except ClientException as exc:
@@ -187,7 +181,7 @@ def run(client_args, other_args, action_file, dry_run=False):
         msg = f'Trying Action ID: {idx}, "{action_def.action}": {action_def.description}'
         try:
             logger.info(msg)
-            process_action(client, action_def, dry_run=dry_run)
+            process_action(client, action_def, dry_run=ctx.params['dry_run'])
         # pylint: disable=broad-except
         except Exception as err:
             exception_handler(action_def, err)
@@ -196,39 +190,16 @@ def run(client_args, other_args, action_file, dry_run=False):
 
 # pylint: disable=unused-argument, redefined-builtin, too-many-arguments, too-many-locals, line-too-long
 @click.command(context_settings=context_settings(), epilog=footer(__version__, tail='command-line.html'))
-@click_opt_wrap(*cli_opts('config'))
-@click_opt_wrap(*cli_opts('hosts'))
-@click_opt_wrap(*cli_opts('cloud_id'))
-@click_opt_wrap(*cli_opts('api_token'))
-@click_opt_wrap(*cli_opts('id'))
-@click_opt_wrap(*cli_opts('api_key'))
-@click_opt_wrap(*cli_opts('username'))
-@click_opt_wrap(*cli_opts('password'))
-@click_opt_wrap(*cli_opts('bearer_auth'))
-@click_opt_wrap(*cli_opts('opaque_id'))
-@click_opt_wrap(*cli_opts('request_timeout'))
-@click_opt_wrap(*cli_opts('http_compress', onoff=ONOFF))
-@click_opt_wrap(*cli_opts('verify_certs', onoff=ONOFF))
-@click_opt_wrap(*cli_opts('ca_certs'))
-@click_opt_wrap(*cli_opts('client_cert'))
-@click_opt_wrap(*cli_opts('client_key'))
-@click_opt_wrap(*cli_opts('ssl_assert_hostname'))
-@click_opt_wrap(*cli_opts('ssl_assert_fingerprint'))
-@click_opt_wrap(*cli_opts('ssl_version'))
-@click_opt_wrap(*cli_opts('master-only', onoff=ONOFF))
-@click_opt_wrap(*cli_opts('skip_version_test', onoff=ONOFF))
+@options_from_dict(OPTION_DEFAULTS)
 @click_opt_wrap(*cli_opts('dry-run', settings=CLICK_DRYRUN))
-@click_opt_wrap(*cli_opts('loglevel', settings=LOGGING_SETTINGS))
-@click_opt_wrap(*cli_opts('logfile', settings=LOGGING_SETTINGS))
-@click_opt_wrap(*cli_opts('logformat', settings=LOGGING_SETTINGS))
 @click.argument('action_file', type=click.Path(exists=True), nargs=1)
 @click.version_option(__version__, '-v', '--version', prog_name="curator")
 @click.pass_context
 def cli(
     ctx, config, hosts, cloud_id, api_token, id, api_key, username, password, bearer_auth,
     opaque_id, request_timeout, http_compress, verify_certs, ca_certs, client_cert, client_key,
     ssl_assert_hostname, ssl_assert_fingerprint, ssl_version, master_only, skip_version_test,
-    dry_run, loglevel, logfile, logformat, action_file
+    loglevel, logfile, logformat, blacklist, dry_run, action_file
 ):
     """
     Curator for Elasticsearch indices
@@ -244,8 +215,8 @@ def cli(
         curator_cli -h
     """
     ctx.obj = {}
-    ctx.obj['dry_run'] = dry_run
-    cfg = get_config(ctx.params, default_config_file())
-    configure_logging(cfg, ctx.params)
-    client_args, other_args = get_args(ctx.params, cfg)
-    run(client_args, other_args, action_file, dry_run)
+    ctx.obj['default_config'] = default_config_file()
+    get_config(ctx)
+    configure_logging(ctx)
+    generate_configdict(ctx)
+    run(ctx)

curator/cli_singletons/alias.py

@@ -1,11 +1,10 @@
 """Alias Singleton"""
 import logging
 import click
-from es_client.helpers.config import context_settings
 from curator.cli_singletons.object_class import CLIAction
 from curator.cli_singletons.utils import json_to_dict, validate_filter_json
 
-@click.command(context_settings=context_settings())
+@click.command()
 @click.option('--name', type=str, help='Alias name', required=True)
 @click.option(
     '--add',
@@ -51,7 +50,7 @@ def alias(ctx, name, add, remove, warn_if_no_indices, extra_settings, allow_ilm_
     ignore_empty_list = warn_if_no_indices
     action = CLIAction(
         ctx.info_name,
-        ctx.obj['config'],
+        ctx.obj['configdict'],
         manual_options,
         [], # filter_list is empty in our case
         ignore_empty_list,

curator/cli_singletons/allocation.py

@@ -1,10 +1,9 @@
 """Allocation Singleton"""
 import click
-from es_client.helpers.config import context_settings
 from curator.cli_singletons.object_class import CLIAction
 from curator.cli_singletons.utils import validate_filter_json
 
-@click.command(context_settings=context_settings())
+@click.command()
 @click.option('--key', type=str, required=True, help='Node identification tag')
 @click.option('--value', type=str, default=None, help='Value associated with --key')
 @click.option('--allocation_type', type=click.Choice(['require', 'include', 'exclude']))
@@ -72,5 +71,5 @@ def allocation(
     }
     # ctx.info_name is the name of the function or name specified in @click.command decorator
     action = CLIAction(
-        ctx.info_name, ctx.obj['config'], manual_options, filter_list, ignore_empty_list)
+        ctx.info_name, ctx.obj['configdict'], manual_options, filter_list, ignore_empty_list)
     action.do_singleton_action(dry_run=ctx.obj['dry_run'])

curator/cli_singletons/close.py

@@ -1,10 +1,9 @@
 """Close Singleton"""
 import click
-from es_client.helpers.config import context_settings
 from curator.cli_singletons.object_class import CLIAction
 from curator.cli_singletons.utils import validate_filter_json
 
-@click.command(context_settings=context_settings())
+@click.command()
 @click.option('--delete_aliases', is_flag=True, help='Delete all aliases from indices to be closed')
 @click.option('--skip_flush', is_flag=True, help='Skip flush phase for indices to be closed')
 @click.option(
@@ -36,5 +35,5 @@ def close(ctx, delete_aliases, skip_flush, ignore_empty_list, allow_ilm_indices,
     }
     # ctx.info_name is the name of the function or name specified in @click.command decorator
     action = CLIAction(
-        ctx.info_name, ctx.obj['config'], manual_options, filter_list, ignore_empty_list)
+        ctx.info_name, ctx.obj['configdict'], manual_options, filter_list, ignore_empty_list)
     action.do_singleton_action(dry_run=ctx.obj['dry_run'])

curator/cli_singletons/delete.py

@@ -1,11 +1,10 @@
 """Delete Index and Delete Snapshot Singletons"""
 import click
-from es_client.helpers.config import context_settings
 from curator.cli_singletons.object_class import CLIAction
 from curator.cli_singletons.utils import validate_filter_json
 
 #### Indices ####
-@click.command(context_settings=context_settings())
+@click.command()
 @click.option(
     '--ignore_empty_list',
     is_flag=True,
@@ -31,15 +30,15 @@ def delete_indices(ctx, ignore_empty_list, allow_ilm_indices, filter_list):
     # ctx.info_name is the name of the function or name specified in @click.command decorator
     action = CLIAction(
         'delete_indices',
-        ctx.obj['config'],
+        ctx.obj['configdict'],
         {'allow_ilm_indices':allow_ilm_indices},
         filter_list,
         ignore_empty_list
     )
     action.do_singleton_action(dry_run=ctx.obj['dry_run'])
 
 #### Snapshots ####
-@click.command(context_settings=context_settings())
+@click.command()
 @click.option('--repository', type=str, required=True, help='Snapshot repository name')
 @click.option('--retry_count', type=int, help='Number of times to retry (max 3)')
 @click.option('--retry_interval', type=int, help='Time in seconds between retries')
@@ -76,7 +75,7 @@ def delete_snapshots(
     # ctx.info_name is the name of the function or name specified in @click.command decorator
     action = CLIAction(
         'delete_snapshots',
-        ctx.obj['config'],
+        ctx.obj['configdict'],
         manual_options,
         filter_list,
         ignore_empty_list,

curator/cli_singletons/forcemerge.py

@@ -1,10 +1,9 @@
 """ForceMerge Singleton"""
 import click
-from es_client.helpers.config import context_settings
 from curator.cli_singletons.object_class import CLIAction
 from curator.cli_singletons.utils import validate_filter_json
 
-@click.command(context_settings=context_settings())
+@click.command()
 @click.option(
     '--max_num_segments',
     type=int,
@@ -44,5 +43,5 @@ def forcemerge(ctx, max_num_segments, delay, ignore_empty_list, allow_ilm_indice
     }
     # ctx.info_name is the name of the function or name specified in @click.command decorator
     action = CLIAction(
-        ctx.info_name, ctx.obj['config'], manual_options, filter_list, ignore_empty_list)
+        ctx.info_name, ctx.obj['configdict'], manual_options, filter_list, ignore_empty_list)
     action.do_singleton_action(dry_run=ctx.obj['dry_run'])