Add base64 signed conjoined API Keys (#1676)

* WIP 20230412

* Not doing hot2cold or hot2frozen right now.

All other changes ready.

CONTRIBUTING.md

@@ -16,7 +16,7 @@ Want to write your own code to do something Curator doesn't do out of the box?
 Curator ships with both an API and wrapper scripts (which are actually defined
 as entry points).  This allows you to write your own scripts to accomplish
 similar goals, or even new and different things with the
-[Curator API](http://curator.readthedocs.io/), [es_client](http://esclient.readthedocs.io), and the
+[Curator API](http://curator.readthedocs.io/), [es_client](https://es-client.readthedocs.io), and the
 [Elasticsearch Python Client Library](http://elasticsearch-py.readthedocs.io/).
 
 Want to know how to use the command-line interface (CLI)?

curator/_version.py

@@ -1,2 +1,2 @@
 """Curator Version"""
-__version__ = '8.0.3'
+__version__ = '8.0.4'

curator/actions/cold2frozen.py

@@ -1,8 +1,7 @@
 """Snapshot and Restore action classes"""
 import logging
-import re
-from curator.helpers.getters import get_data_tiers
-from curator.helpers.testers import verify_index_list
+from curator.helpers.getters import get_alias_actions, get_frozen_prefix, get_tier_preference
+from curator.helpers.testers import has_lifecycle_name, is_idx_partial, verify_index_list
 from curator.helpers.utils import report_failure
 from curator.exceptions import CuratorException, FailedExecution, SearchableSnapshotException
 
@@ -69,107 +68,6 @@ def assign_kwargs(self, **kwargs):
             else:
                 setattr(self, key, value)
 
-    def get_alias_actions(self, oldidx, newidx, aliases):
-        """
-        :param oldidx: The old index name
-        :param newidx: The new index name
-        :param aliases: The aliases
-
-        :type oldidx: str
-        :type newidx: str
-        :type aliases: dict
-
-        :returns: A list of actions suitable for
-            :py:meth:`~.elasticsearch.client.IndicesClient.update_aliases` ``actions`` kwarg.
-        :rtype: list
-        """
-        actions = []
-        for alias in aliases.keys():
-            actions.append({'remove': {'index': oldidx, 'alias': alias}})
-            actions.append({'add': {'index': newidx, 'alias': alias}})
-        return actions
-
-    def get_frozen_prefix(self, oldidx, curridx):
-        """
-        Use regular expression magic to extract the prefix from the current index, and then use
-        that with ``partial-`` in front to name the resulting index.
-
-        If there is no prefix, then we just send back ``partial-``
-
-        :param oldidx: The index name before it was mounted in cold tier
-        :param curridx: The current name of the index, as mounted in cold tier
-
-        :type oldidx: str
-        :type curridx: str
-
-        :returns: The prefix to prepend the index name with for mounting as frozen
-        :rtype: str
-        """
-        pattern = f'^(.*){oldidx}$'
-        regexp = re.compile(pattern)
-        match = regexp.match(curridx)
-        prefix = match.group(1)
-        self.loggit.debug('Detected match group for prefix: %s', prefix)
-        if not prefix:
-            return 'partial-'
-        return f'partial-{prefix}'
-
-    def get_tier_preference(self):
-        """Do the tier preference thing in reverse order from coldest to hottest
-
-        :returns: A suitable tier preference string in csv format
-        :rtype: str
-        """
-        tiers = get_data_tiers(self.client)
-        # We're migrating from cold to frozen here. If a frozen tier exists, frozen ss mounts
-        # should only ever go to the frozen tier.
-        if 'data_frozen' in tiers and tiers['data_frozen']:
-            return 'data_frozen'
-        # If there are no  nodes with the 'data_frozen' role...
-        preflist = []
-        for key in ['data_cold', 'data_warm', 'data_hot']:
-            # This ordering ensures that colder tiers are prioritized
-            if key in tiers and tiers[key]:
-                preflist.append(key)
-        # If all of these are false, then we have no data tiers and must use 'data_content'
-        if not preflist:
-            return 'data_content'
-        # This will join from coldest to hottest as csv string, e.g. 'data_cold,data_warm,data_hot'
-        return ','.join(preflist)
-
-    def has_lifecycle_name(self, idx_settings):
-        """
-        :param idx_settings: The settings for an index being tested
-        :type idx_settings: dict
-
-        :returns: ``True`` if a lifecycle name exists in settings, else ``False``
-        :rtype: bool
-        """
-        if 'lifecycle' in idx_settings:
-            if 'name' in idx_settings['lifecycle']:
-                return True
-        return False
-
-    def is_idx_partial(self, idx_settings):
-        """
-        :param idx_settings: The settings for an index being tested
-        :type idx_settings: dict
-
-        :returns: ``True`` if store.snapshot.partial exists in settings, else ``False``
-        :rtype: bool
-        """
-        if 'store' in idx_settings:
-            if 'snapshot' in idx_settings['store']:
-                if 'partial' in idx_settings['store']['snapshot']:
-                    if idx_settings['store']['snapshot']['partial']:
-                        return True
-                    # store.snapshot.partial exists but is False -- Not a frozen tier mount
-                    return False
-                # store.snapshot exists, but partial isn't there -- Possibly a cold tier mount
-                return False
-            raise SearchableSnapshotException('Index not a mounted searchable snapshot')
-        raise SearchableSnapshotException('Index not a mounted searchable snapshot')
-
     def action_generator(self):
         """Yield a dict for use in :py:meth:`do_action` and :py:meth:`do_dry_run`
 
@@ -179,28 +77,28 @@ def action_generator(self):
         """
         for idx in self.index_list.indices:
             idx_settings = self.client.indices.get(index=idx)[idx]['settings']['index']
-            if self.has_lifecycle_name(idx_settings):
+            if has_lifecycle_name(idx_settings):
                 self.loggit.critical(
                     'Index %s is associated with an ILM policy and this action will never work on '
                     'an index associated with an ILM policy', idx)
                 raise CuratorException(f'Index {idx} is associated with an ILM policy')
-            if self.is_idx_partial(idx_settings):
+            if is_idx_partial(idx_settings):
                 self.loggit.critical('Index %s is already in the frozen tier', idx)
                 raise SearchableSnapshotException('Index is already in frozen tier')
             snap = idx_settings['store']['snapshot']['snapshot_name']
             snap_idx = idx_settings['store']['snapshot']['index_name']
             repo = idx_settings['store']['snapshot']['repository_name']
             aliases = self.client.indices.get(index=idx)[idx]['aliases']
 
-            prefix = self.get_frozen_prefix(snap_idx, idx)
+            prefix = get_frozen_prefix(snap_idx, idx)
             renamed = f'{prefix}{snap_idx}'
 
             if not self.index_settings:
                 self.index_settings = {
                     "routing": {
                         "allocation": {
                             "include": {
-                                "_tier_preference": self.get_tier_preference()
+                                "_tier_preference": get_tier_preference(self.client)
                             }
                         }
                     }
@@ -255,7 +153,7 @@ def do_action(self):
                 # Verify it's mounted as a partial now:
                 self.loggit.debug('Verifying new index %s is mounted properly...', newidx)
                 idx_settings = self.client.indices.get(index=newidx)[newidx]
-                if self.is_idx_partial(idx_settings['settings']['index']):
+                if is_idx_partial(idx_settings['settings']['index']):
                     self.loggit.info('Index %s is mounted for frozen tier', newidx)
                 else:
                     raise SearchableSnapshotException(
@@ -267,7 +165,7 @@ def do_action(self):
                 else:
                     self.loggit.debug('Transferring aliases to new index %s', newidx)
                     self.client.indices.update_aliases(
-                        actions=self.get_alias_actions(current_idx, newidx, aliases))
+                        actions=get_alias_actions(current_idx, newidx, aliases))
                     verify = self.client.indices.get(index=newidx)[newidx]['aliases'].keys()
                     if alias_names != verify:
                         self.loggit.error(

curator/actions/create_index.py

@@ -64,10 +64,13 @@ def do_action(self):
         :py:meth:`~.elasticsearch.client.IndicesClient.create` index identified by :py:attr:`name`
         with values from :py:attr:`aliases`, :py:attr:`mappings`, and :py:attr:`settings`
         """
-        msg = (f'Creating index "{self.name}" with settings: {self.extra_settings}')
+        msg = f'Creating index "{self.name}" with settings: {self.extra_settings}'
         self.loggit.info(msg)
         try:
-            self.client.indices.create(index=self.name, aliases=self.aliases, mappings=self.mappings, settings=self.settings)
+            self.client.indices.create(
+                index=self.name, aliases=self.aliases, mappings=self.mappings,
+                settings=self.settings
+            )
         # Most likely error is a 400, `resource_already_exists_exception`
         except RequestError as err:
             match_list = ["index_already_exists_exception", "resource_already_exists_exception"]

curator/cli.py

@@ -241,6 +241,7 @@ def run(client_args, other_args, action_file, dry_run=False):
 @click.option('--config', help='Path to configuration file.', type=click.Path(exists=True), default=settings.config_file())
 @click.option('--hosts', help='Elasticsearch URL to connect to', multiple=True)
 @click.option('--cloud_id', help='Shorthand to connect to Elastic Cloud instance')
+@click.option('--api_token', help='The base64 encoded API Key token', type=str)
 @click.option('--id', help='API Key "id" value', type=str)
 @click.option('--api_key', help='API Key "api_key" value', type=str)
 @click.option('--username', help='Username used to create "basic_auth" tuple')
@@ -266,7 +267,7 @@ def run(client_args, other_args, action_file, dry_run=False):
 @click.version_option(version=__version__)
 @click.pass_context
 def cli(
-    ctx, config, hosts, cloud_id, id, api_key, username, password, bearer_auth,
+    ctx, config, hosts, cloud_id, api_token, id, api_key, username, password, bearer_auth,
     opaque_id, request_timeout, http_compress, verify_certs, ca_certs, client_cert, client_key,
     ssl_assert_hostname, ssl_assert_fingerprint, ssl_version, master_only, skip_version_test,
     dry_run, loglevel, logfile, logformat, action_file
@@ -312,11 +313,12 @@ def cli(
         'password': password,
         'api_key': {
             'id': id,
-            'api_key': api_key
+            'api_key': api_key,
+            'token': api_token,
         }
     })
-    # Remove `api_key` root key if `id` and `api_key` are both None
-    if id is None and api_key is None:
+    # Remove `api_key` root key if `id` and `api_key` and `token` are all None
+    if id is None and api_key is None and api_token is None:
         del cli_other['api_key']
 
     # If hosts are in the config file, but cloud_id is specified at the command-line,

curator/helpers/getters.py

@@ -1,6 +1,7 @@
 """Utility functions that get things"""
 # :pylint disable=
 import logging
+import re
 from elasticsearch8 import exceptions as es8exc
 from es_client.defaults import VERSION_MAX, VERSION_MIN
 from es_client.builder import Builder
@@ -24,6 +25,26 @@ def byte_size(num, suffix='B'):
         num /= 1024.0
     return f'{num:.1f}Y{suffix}'
 
+def get_alias_actions(oldidx, newidx, aliases):
+    """
+    :param oldidx: The old index name
+    :param newidx: The new index name
+    :param aliases: The aliases
+
+    :type oldidx: str
+    :type newidx: str
+    :type aliases: dict
+
+    :returns: A list of actions suitable for
+        :py:meth:`~.elasticsearch.client.IndicesClient.update_aliases` ``actions`` kwarg.
+    :rtype: list
+    """
+    actions = []
+    for alias in aliases.keys():
+        actions.append({'remove': {'index': oldidx, 'alias': alias}})
+        actions.append({'add': {'index': newidx, 'alias': alias}})
+    return actions
+
 def get_client(
     configdict=None, configfile=None, autoconnect=False, version_min=VERSION_MIN,
     version_max=VERSION_MAX):
@@ -87,6 +108,32 @@ def role_check(role, node_info):
                 retval[role] = True
     return retval
 
+def get_frozen_prefix(oldidx, curridx):
+    """
+    Use regular expression magic to extract the prefix from the current index, and then use
+    that with ``partial-`` in front to name the resulting index.
+
+    If there is no prefix, then we just send back ``partial-``
+
+    :param oldidx: The index name before it was mounted in cold tier
+    :param curridx: The current name of the index, as mounted in cold tier
+
+    :type oldidx: str
+    :type curridx: str
+
+    :returns: The prefix to prepend the index name with for mounting as frozen
+    :rtype: str
+    """
+    logger = logging.getLogger(__name__)
+    pattern = f'^(.*){oldidx}$'
+    regexp = re.compile(pattern)
+    match = regexp.match(curridx)
+    prefix = match.group(1)
+    logger.debug('Detected match group for prefix: %s', prefix)
+    if not prefix:
+        return 'partial-'
+    return f'partial-{prefix}'
+
 def get_indices(client):
     """
     Calls :py:meth:`~.elasticsearch.client.IndicesClient.get_settings`
@@ -181,6 +228,48 @@ def get_snapshot_data(client, repository=None):
         )
         raise FailedExecution(msg) from err
 
+def get_tier_preference(client, target_tier='data_frozen'):
+    """Do the tier preference thing in reverse order from coldest to hottest
+    Based on the value of ``target_tier``, build out the list to use.
+
+    :param client: A client connection object
+    :param target_tier: The target data tier, e.g. data_warm.
+
+    :type client: :py:class:`~.elasticsearch.Elasticsearch`
+    :type target_tier: str
+
+    :returns: A suitable tier preference string in csv format
+    :rtype: str
+    """
+    tiermap = {
+        'data_content': 0,
+        'data_hot': 1,
+        'data_warm': 2,
+        'data_cold': 3,
+        'data_frozen': 4,
+    }
+    tiers = get_data_tiers(client)
+    test_list = []
+    for tier in ['data_hot', 'data_warm', 'data_cold', 'data_frozen']:
+        if tier in tiers and tiermap[tier] <= tiermap[target_tier]:
+            test_list.insert(0, tier)
+    if target_tier == 'data_frozen':
+        # We're migrating to frozen here. If a frozen tier exists, frozen searchable snapshot
+        # mounts should only ever go to the frozen tier.
+        if 'data_frozen' in tiers and tiers['data_frozen']:
+            return 'data_frozen'
+    # If there are no  nodes with the 'data_frozen' role...
+    preflist = []
+    for key in test_list:
+        # This ordering ensures that colder tiers are prioritized
+        if key in tiers and tiers[key]:
+            preflist.append(key)
+    # If all of these are false, then we have no data tiers and must use 'data_content'
+    if not preflist:
+        return 'data_content'
+    # This will join from coldest to hottest as csv string, e.g. 'data_cold,data_warm,data_hot'
+    return ','.join(preflist)
+
 def get_write_index(client, alias):
     """
     Calls :py:meth:`~.elasticsearch.client.IndicesClient.get_alias`