Merge branch 'master' into 8.x

curator/_version.py

@@ -1,2 +1,2 @@
 """Curator Version"""
-__version__ = '8.0.5'
+__version__ = '8.0.6'

curator/indexlist.py

@@ -72,6 +72,7 @@ def __get_indices(self):
         if self.indices:
             for index in self.indices:
                 self.__build_index_info(index)
+            self.loggit.debug('FRESH UNPOPULATED: self.index_info = %s', self.index_info)
             self.get_metadata()
             self.get_index_stats()
 
@@ -195,12 +196,38 @@ def _bulk_queries(self, data, exec_func):
     def _get_cluster_state(self, data):
         return self.client.cluster.state(index=to_csv(data), metric='metadata')['metadata']['indices']
 
+    def mitigate_alias(self, index):
+        """
+        Mitigate when an alias is detected instead of an index name
+
+        :param index: The index name that is showing up *instead* of what was expected
+
+        :type index: str
+
+        :returns: No return value:
+        :rtype: None
+        """
+        self.loggit.debug('Correcting an instance where an alias name points to index "%s"', index)
+        data = self.client.indices.get(index=index)
+        aliases = list(data[index]['aliases'])
+        if aliases:
+            for alias in aliases:
+                if alias in self.indices:
+                    self.loggit.warning('Removing alias "%s" from IndexList.indices', alias)
+                    self.indices.remove(alias)
+                if alias in list(self.index_info):
+                    self.loggit.warning('Removing alias "%s" from IndexList.index_info', alias)
+                    del self.index_info[alias]
+        self.loggit.debug('Adding "%s" to IndexList.indices', index)
+        self.indices.append(index)
+        self.loggit.debug('Adding preliminary metadata for "%s" to IndexList.index_info', index)
+        self.__build_index_info(index)
+
     def get_metadata(self):
         """
         Populate ``index_info`` with index ``size_in_bytes`` and doc count information for each
         index.
         """
-        self.loggit.debug('Getting index metadata')
         self.empty_list_check()
         for lst in chunk_index_list(self.indices):
             working_list = {}
@@ -216,8 +243,25 @@ def get_metadata(self):
                     working_list = {}
                     working_list.update(self._bulk_queries(lst, self._get_cluster_state))
             if working_list:
+                self.loggit.debug('working_list.keys() = %s', working_list.keys())
                 for index in list(working_list.keys()):
-                    sii = self.index_info[index]
+                    self.loggit.debug('index = %s', index)
+                    try:
+                        sii = self.index_info[index]
+                    except KeyError:
+                        # What I believe has happened with this race condition is that during
+                        # __build_index_info initially, an index has a name, but is in process
+                        # of being remounted as a searchable snapshot index, and suddenly the
+                        # original index name is an alias, and the _get_cluster_state call returns
+                        # the new, actual index name instead of the alias it had. This is how a
+                        # KeyError can actually happen in this case. The relevant logs showing
+                        # this behavior are in issue #1682
+                        self.loggit.warning(
+                            'Index %s was not present at IndexList initialization, and may be '
+                            'behind an alias', index
+                        )
+                        self.mitigate_alias(index)
+                        sii = self.index_info[index]
                     wli = working_list[index]
                     sii['age']['creation_date'] = (
                         fix_epoch(wli['settings']['index']['creation_date'])

docs/Changelog.rst

@@ -3,6 +3,15 @@
 Changelog
 =========
 
+8.0.6 (18 July 2023)
+--------------------
+
+**Breakfix Release**
+
+  * Small breakfix change to catch a similar rare race condition patched in
+    8.0.5 covering the ``get_index_stats()`` method of IndexList. This patch
+    covers the ``get_metadata()`` method and closes #1682.
+
 8.0.5 (13 July 2023)
 --------------------
 

docs/asciidoc/index.asciidoc

@@ -1,4 +1,4 @@
-:curator_version: 8.0.5
+:curator_version: 8.0.6
 :curator_major: 8
 :curator_doc_tree: 8.0
 :es_py_version: 8.8.2

tests/integration/test_integrations.py

@@ -97,3 +97,25 @@ def test_get_index_stats_with_404(self):
             # Guarantee we're getting the expected WARNING level message
             assert self._caplog.records[-1].message == expected
         assert ilo.indices == [self.IDX3]
+    def test_get_metadata_with_keyerror(self):
+        """Check to ensure that metadata is being collected if a new index shows up"""
+        expected1 = f'Removing alias "{self.IDX2}" from IndexList.index_info'
+        expected2 = f'Removing alias "{self.IDX2}" from IndexList.indices'
+        expected3 = (
+            f'Index {self.IDX3} was not present at IndexList initialization, '
+            f'and may be behind an alias'
+        )
+        alias = {self.IDX2: {}}
+        self.create_index(self.IDX1)
+        self.create_index(self.IDX2)
+        ilo = IndexList(self.client)
+        assert ilo.indices == [self.IDX1, self.IDX2]
+        self.client.indices.delete(index=self.IDX2)
+        self.client.indices.create(index=self.IDX3, aliases=alias)
+        with self._caplog.at_level(logging.WARNING):
+            ilo.get_metadata()
+            # Guarantee we're getting the expected WARNING level messages
+            assert self._caplog.records[-3].message == expected3
+            assert self._caplog.records[-2].message == expected2
+            assert self._caplog.records[-1].message == expected1
+        assert ilo.indices == [self.IDX1, self.IDX3]