Update CLI to use same flags for command-line (#1666)

...as the singleton CLI

This should be great for Docker users to pass values as environment variables

curator/_version.py

@@ -1,2 +1,2 @@
 """Curator Version"""
-__version__ = '8.0.1'
+__version__ = '8.0.2'

curator/cli.py

@@ -2,8 +2,8 @@
 import sys
 import logging
 import click
-from es_client.builder import ClientArgs, OtherArgs, Builder
-from es_client.helpers.utils import get_yaml, check_config, prune_nones
+from es_client.builder import ClientArgs, OtherArgs
+from es_client.helpers.utils import get_yaml, check_config, prune_nones, verify_url_schema
 from curator.actions import (
     Alias, Allocation, Close, ClusterRouting, CreateIndex, DeleteIndices, ForceMerge,
     IndexSettings, Open, Reindex, Replicas, Rollover, Shrink, Snapshot, DeleteSnapshots, Restore
@@ -12,10 +12,11 @@
 from curator.config_utils import check_logging_config, password_filter, set_logging
 from curator.defaults import settings
 from curator.exceptions import NoIndices, NoSnapshots
-from curator.helpers.getters import get_write_index
-from curator.helpers.testers import validate_actions
+from curator.helpers.getters import get_client
+from curator.helpers.testers import ilm_policy_check, validate_actions
 from curator.indexlist import IndexList
 from curator.snapshotlist import SnapshotList
+from curator.cli_singletons.utils import get_width
 from curator._version import __version__
 
 CLASS_MAP = {
@@ -37,6 +38,50 @@
     'shrink' : Shrink,
 }
 
+def override_logging(config, loglevel, logfile, logformat):
+    """Get logging config and override from command-line options
+
+    :param config: The configuration from file
+    :param loglevel: The log level
+    :param logfile: The log file to write
+    :param logformat: Which log format to use
+
+    :type config: dict
+    :type loglevel: str
+    :type logfile: str
+    :type logformat: str
+
+    :returns: Log configuration ready for validation
+    :rtype: dict
+    """
+    # Check for log settings from config file
+    init_logcfg = check_logging_config(config)
+
+    # Override anything with options from the command-line
+    if loglevel:
+        init_logcfg['loglevel'] = loglevel
+    if logfile:
+        init_logcfg['logfile'] = logfile
+    if logformat:
+        init_logcfg['logformat'] = logformat
+    return init_logcfg
+
+def cli_hostslist(hosts):
+    """
+    :param hosts: One or more hosts.
+    :type hosts: str or list
+
+    :returns: A list of hosts that came in from the command-line, or ``None``
+    :rtype: list or ``None``
+    """
+    hostslist = []
+    if hosts:
+        for host in list(hosts):
+            hostslist.append(verify_url_schema(host))
+    else:
+        hostslist = None
+    return hostslist
+
 def process_action(client, config, **kwargs):
     """
     Do the ``action`` in ``config``, using the associated options and ``kwargs``, if any.
@@ -106,38 +151,21 @@ def process_action(client, config, **kwargs):
         logger.debug('Doing the action here.')
         action_obj.do_action()
 
-def run(config, action_file, dry_run=False):
+def run(client_args, other_args, action_file, dry_run=False):
     """
     Called by :py:func:`cli` to execute what was collected at the command-line
+
+    :param client_args: The ClientArgs arguments object
+    :param other_args: The OtherArgs arguments object
+    :param action_file: The action configuration file
+    :param dry_run: Do not perform any changes
+
+    :type client_args: :py:class:`~.es_client.ClientArgs`
+    :type other_args: :py:class:`~.es_client.OtherArgs`
+    :type action_file: str
+    :type dry_run: bool
     """
-    # """Process yaml_file and return a valid client configuration"""
-    config_dict = get_yaml(config)
-    if config_dict is None:
-        click.echo('Empty configuration file provided. Using defaults')
-        config_dict = {}
-    elif not isinstance(config_dict, dict):
-        raise ConfigurationError('Configuration file not converted to dictionary. Check YAML configuration.')
-    set_logging(check_logging_config(config_dict))
-    # set_logging({'loglevel':'DEBUG','blacklist':[]})
     logger = logging.getLogger(__name__)
-    if not isinstance(config_dict, dict):
-        config_dict = {}
-        logger.warning(
-            'Provided config file %s was unable to be properly read, or is empty. '
-            'Using empty dictionary (assuming defaults)', config)
-    logger.debug('config_dict = %s', config_dict)
-    client_args = ClientArgs()
-    other_args = OtherArgs()
-    if config:
-        raw_config = check_config(config_dict)
-        logger.debug('raw_config = %s', raw_config)
-        try:
-            client_args.update_settings(raw_config['client'])
-        # pylint: disable=broad-except
-        except Exception as exc:
-            click.echo(f'EXCEPTION = {exc}')
-            sys.exit(1)
-        other_args.update_settings(raw_config['other_settings'])
 
     logger.debug('Client and logging configuration options validated.')
 
@@ -189,33 +217,28 @@ def run(config, action_file, dry_run=False):
         # Create a client object for each action...
         logger.info('Creating client object and testing connection')
 
-        # Build a "final_config" that reflects CLI args overriding anything from a config_file
-        final_config = {
-            'elasticsearch': {
-                'client': prune_nones(client_args.asdict()),
-                'other_settings': prune_nones(other_args.asdict())
-            }
-        }
-        builder = Builder(configdict=final_config)
-
         try:
-            builder.connect()
-        except Exception as exc:
-            click.echo(f'Exception encountered: {exc}')
-            raise ClientException from exc
+            client = get_client(configdict={
+                'elasticsearch': {
+                    'client': prune_nones(client_args.asdict()),
+                    'other_settings': prune_nones(other_args.asdict())
+                }
+            })
+        except ClientException as exc:
+            # No matter where logging is set to go, make sure we dump these messages to the CLI
+            click.echo('Unable to establish client connection to Elasticsearch!')
+            click.echo(f'Exception: {exc}')
+            sys.exit(1)
 
-        client = builder.client
         ### Filter ILM indices unless expressly permitted
         if allow_ilm:
             logger.warning('allow_ilm_indices: true')
             logger.warning('Permitting operation on indices with an ILM policy')
         if not allow_ilm and action not in settings.snapshot_actions():
             if actions_config[idx]['action'] == 'rollover':
                 alias = actions_config[idx]['options']['name']
-                write_index = get_write_index(client, alias)
                 try:
-                    idx_settings = client.indices.get_settings(index=write_index)
-                    if 'name' in idx_settings[write_index]['settings']['index']['lifecycle']:
+                    if ilm_policy_check(client, alias):
                         logger.info('Alias %s is associated with ILM policy.', alias)
                         logger.info('Skipping action %s because allow_ilm_indices is false.', idx)
                         continue
@@ -235,7 +258,7 @@ def run(config, action_file, dry_run=False):
             process_action(client, actions_config[idx], **kwargs)
         # pylint: disable=broad-except
         except Exception as err:
-            if isinstance(err, NoIndices) or isinstance(err, NoSnapshots):
+            if isinstance(err, (NoIndices, NoSnapshots)):
                 if ignore_empty_list:
                     logger.info('Skipping action "%s" due to empty list: %s', action, type(err))
                 else:
@@ -250,19 +273,106 @@ def run(config, action_file, dry_run=False):
         logger.info('Action ID: %s, "%s" completed.', idx, action)
     logger.info('Job completed.')
 
-@click.command()
-@click.option(
-    '--config',
-    help="Path to configuration file. Default: ~/.curator/curator.yml",
-    type=click.Path(exists=True), default=settings.config_file()
-)
+@click.command(context_settings=get_width())
+@click.option('--config', help='Path to configuration file.', type=click.Path(exists=True), default=settings.config_file())
+@click.option('--hosts', help='Elasticsearch URL to connect to', multiple=True)
+@click.option('--cloud_id', help='Shorthand to connect to Elastic Cloud instance')
+@click.option('--id', help='API Key "id" value', type=str)
+@click.option('--api_key', help='API Key "api_key" value', type=str)
+@click.option('--username', help='Username used to create "basic_auth" tuple')
+@click.option('--password', help='Password used to create "basic_auth" tuple')
+@click.option('--bearer_auth', type=str)
+@click.option('--opaque_id', type=str)
+@click.option('--request_timeout', help='Request timeout in seconds', type=float)
+@click.option('--http_compress', help='Enable HTTP compression', is_flag=True, default=None)
+@click.option('--verify_certs', help='Verify SSL/TLS certificate(s)', is_flag=True, default=None)
+@click.option('--ca_certs', help='Path to CA certificate file or directory')
+@click.option('--client_cert', help='Path to client certificate file')
+@click.option('--client_key', help='Path to client certificate key')
+@click.option('--ssl_assert_hostname', help='Hostname or IP address to verify on the node\'s certificate.', type=str)
+@click.option('--ssl_assert_fingerprint', help='SHA-256 fingerprint of the node\'s certificate. If this value is given then root-of-trust verification isn\'t done and only the node\'s certificate fingerprint is verified.', type=str)
+@click.option('--ssl_version', help='Minimum acceptable TLS/SSL version', type=str)
+@click.option('--master-only', help='Only run if the single host provided is the elected master', is_flag=True, default=None)
+@click.option('--skip_version_test', help='Do not check the host version', is_flag=True, default=None)
 @click.option('--dry-run', is_flag=True, help='Do not perform any changes.')
+@click.option('--loglevel', help='Log level', type=click.Choice(['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']))
+@click.option('--logfile', help='log file')
+@click.option('--logformat', help='Log output format', type=click.Choice(['default', 'logstash', 'json', 'ecs']))
 @click.argument('action_file', type=click.Path(exists=True), nargs=1)
 @click.version_option(version=__version__)
-def cli(config, dry_run, action_file):
+@click.pass_context
+def cli(
+    ctx, config, hosts, cloud_id, id, api_key, username, password, bearer_auth,
+    opaque_id, request_timeout, http_compress, verify_certs, ca_certs, client_cert, client_key,
+    ssl_assert_hostname, ssl_assert_fingerprint, ssl_version, master_only, skip_version_test,
+    dry_run, loglevel, logfile, logformat, action_file
+):
     """
     Curator for Elasticsearch indices.
 
     See http://elastic.co/guide/en/elasticsearch/client/curator/current
     """
-    run(config, action_file, dry_run)
+    client_args = ClientArgs()
+    other_args = OtherArgs()
+    if config:
+        from_yaml = get_yaml(config)
+        raw_config = check_config(from_yaml)
+        client_args.update_settings(raw_config['client'])
+        other_args.update_settings(raw_config['other_settings'])
+
+    set_logging(check_logging_config(
+        {'logging': override_logging(from_yaml, loglevel, logfile, logformat)}))
+
+    hostslist = cli_hostslist(hosts)
+
+    cli_client = prune_nones({
+        'hosts': hostslist,
+        'cloud_id': cloud_id,
+        'bearer_auth': bearer_auth,
+        'opaque_id': opaque_id,
+        'request_timeout': request_timeout,
+        'http_compress': http_compress,
+        'verify_certs': verify_certs,
+        'ca_certs': ca_certs,
+        'client_cert': client_cert,
+        'client_key': client_key,
+        'ssl_assert_hostname': ssl_assert_hostname,
+        'ssl_assert_fingerprint': ssl_assert_fingerprint,
+        'ssl_version': ssl_version
+    })
+
+    cli_other = prune_nones({
+        'master_only': master_only,
+        'skip_version_test': skip_version_test,
+        'username': username,
+        'password': password,
+        'api_key': {
+            'id': id,
+            'api_key': api_key
+        }
+    })
+    # Remove `api_key` root key if `id` and `api_key` are both None
+    if id is None and api_key is None:
+        del cli_other['api_key']
+
+    # If hosts are in the config file, but cloud_id is specified at the command-line,
+    # we need to remove the hosts parameter as cloud_id and hosts are mutually exclusive
+    if cloud_id:
+        click.echo('cloud_id provided at CLI, superseding any other configured hosts')
+        client_args.hosts = None
+        cli_client.pop('hosts', None)
+
+    # Likewise, if hosts are provided at the command-line, but cloud_id was in the config file,
+    # we need to remove the cloud_id parameter from the config file-based dictionary before merging
+    if hosts:
+        click.echo('hosts specified manually, superseding any other cloud_id or hosts')
+        client_args.hosts = None
+        client_args.cloud_id = None
+        cli_client.pop('cloud_id', None)
+
+    # Update the objects if we have settings after pruning None values
+    if cli_client:
+        client_args.update_settings(cli_client)
+    if cli_other:
+        other_args.update_settings(cli_other)
+    run(client_args, other_args, action_file, dry_run)

curator/helpers/getters.py

@@ -2,7 +2,9 @@
 # :pylint disable=
 import logging
 from elasticsearch8 import exceptions as es8exc
-from curator.exceptions import CuratorException, FailedExecution, MissingArgument
+from es_client.defaults import VERSION_MAX, VERSION_MIN
+from es_client.builder import Builder
+from curator.exceptions import ClientException, CuratorException, FailedExecution, MissingArgument
 
 def byte_size(num, suffix='B'):
     """
@@ -22,6 +24,46 @@ def byte_size(num, suffix='B'):
         num /= 1024.0
     return f'{num:.1f}Y{suffix}'
 
+def get_client(
+    configdict=None, configfile=None, autoconnect=False, version_min=VERSION_MIN,
+    version_max=VERSION_MAX):
+    """Get an Elasticsearch Client using :py:class:`es_client.Builder`
+
+    Build a client out of settings from `configfile` or `configdict`
+    If neither `configfile` nor `configdict` is provided, empty defaults will be used.
+    If both are provided, `configdict` will be used, and `configfile` ignored.
+
+    :param configdict: A configuration dictionary
+    :param configfile: A configuration file
+    :param autoconnect: Connect to client automatically
+    :param verion_min: Minimum acceptable version of Elasticsearch (major, minor, patch)
+    :param verion_max: Maximum acceptable version of Elasticsearch (major, minor, patch)
+
+    :type configdict: dict
+    :type configfile: str
+    :type autoconnect: bool
+    :type version_min: tuple
+    :type version_max: tuple
+
+    :returns: A client connection object
+    :rtype: :py:class:`~.elasticsearch.Elasticsearch`
+    """
+    logger = logging.getLogger(__name__)
+    logger.info('Creating client object and testing connection')
+
+    builder = Builder(
+        configdict=configdict, configfile=configfile, autoconnect=autoconnect,
+        version_min=version_min, version_max=version_max
+    )
+
+    try:
+        builder.connect()
+    except Exception as exc:
+        logger.critical('Exception encountered: %s', exc)
+        raise ClientException from exc
+
+    return builder.client
+
 def get_indices(client):
     """
     Calls :py:meth:`~.elasticsearch.client.IndicesClient.get_settings`

curator/helpers/testers.py

@@ -4,13 +4,38 @@
 from elasticsearch8 import Elasticsearch
 from elasticsearch8.exceptions import NotFoundError
 from es_client.helpers.utils import prune_nones
-from curator.helpers.getters import get_repository
+from curator.helpers.getters import get_repository, get_write_index
 from curator.exceptions import ConfigurationError, MissingArgument, RepositoryException
 from curator.defaults.settings import index_filtertypes, snapshot_actions, snapshot_filtertypes
 from curator.validators import SchemaCheck, actions, options
 from curator.validators.filter_functions import validfilters
 from curator.helpers.utils import report_failure
 
+def ilm_policy_check(client, alias):
+    """Test if alias is associated with an ILM policy
+
+    Calls :py:meth:`~.elasticsearch.client.IndicesClient.get_settings`
+
+    :param client: A client connection object
+    :param alias: The alias name
+
+    :type client: :py:class:`~.elasticsearch.Elasticsearch`
+    :type alias: str
+    :rtype: bool
+    """
+    logger = logging.getLogger(__name__)
+    # alias = action_obj.options['name']
+    write_index = get_write_index(client, alias)
+    try:
+        idx_settings = client.indices.get_settings(index=write_index)
+        if 'name' in idx_settings[write_index]['settings']['index']['lifecycle']:
+            # logger.info('Alias %s is associated with ILM policy.', alias)
+            # logger.info('Skipping action %s because allow_ilm_indices is false.', idx)
+            return True
+    except KeyError:
+        logger.debug('No ILM policies associated with %s', alias)
+    return False
+
 def repository_exists(client, repository=None):
     """
     Calls :py:meth:`~.elasticsearch.client.SnapshotClient.get_repository`