Merge branch 'master' into 8.x

curator/_version.py

@@ -1,2 +1,2 @@
 """Curator Version"""
-__version__ = '8.0.7'
+__version__ = '8.0.8'

curator/helpers/utils.py

@@ -69,7 +69,12 @@ def show_dry_run(ilo, action, **kwargs):
     logger.info(msg)
     indices = sorted(ilo.indices)
     for idx in indices:
-        index_closed = ilo.index_info[idx]['state'] == 'close'
+        # Dry runs need index state, so we collect it here if it's not present.
+        try:
+            index_closed = ilo.index_info[idx]['state'] == 'close'
+        except KeyError:
+            ilo.get_index_state()
+            index_closed = ilo.index_info[idx]['state'] == 'close'
         var = ' (CLOSED)' if index_closed else ''
         msg = f'DRY-RUN: {action}: {idx}{var} with arguments: {kwargs}'
         logger.info(msg)

curator/indexlist.py

@@ -69,11 +69,9 @@ def __get_indices(self):
         self.loggit.debug('Getting all indices')
         self.all_indices = get_indices(self.client)
         self.indices = self.all_indices[:]
-        if self.indices:
-            for index in self.indices:
-                self.__build_index_info(index)
-            # self.get_index_settings()
-            # self.get_index_stats()
+        # if self.indices:
+        #     for index in self.indices:
+        #         self.__build_index_info(index)
 
     def __build_index_info(self, index):
         """
@@ -129,10 +127,6 @@ def __zero_values(self):
             'state' : '',
         }
 
-    # def _get_cluster_state(self, data):
-    #     return self.client.cluster.state(
-    #         index=to_csv(data), metric='metadata')['metadata']['indices']
-
     def _get_indices_segments(self, data):
         return self.client.indices.segments(index=to_csv(data))['indices'].copy()
 

docs/Changelog.rst

@@ -3,6 +3,31 @@
 Changelog
 =========
 
+8.0.8 (21 July 2023)
+--------------------
+
+**Announcements**
+
+Small change to further reduce memory usage by not creating unused data
+structures.
+
+This revealed a glitch with dry-runs that would eventually have been reported.
+
+**Changes**
+
+  * Don't populate IndexList.index_info until required by a filter. In other
+    words, stop populating the zero values as part of instantiation.
+  * This uncovered an oversight with the 8.0.7 release. Certain actions, if
+    taken with no filters, or only a pattern filter, would never ever populate
+    the index_info. This wasn't a huge problem, unless you were using the
+    dry-run flag. The dry-run output requires the index_info to be present for
+    each index. In 8.0.7, where the empty structure was already in place, a
+    dry-run wouldn't fail, but it also wouldn't show any data. This is fixed.
+  * A few tests also needed updating. They were using contrived scenarios to
+    test certain conditions. Now these tests are manually grabbing necessary
+    metadata so they can pass.
+
+
 8.0.7 (21 July 2023)
 --------------------
 

docs/asciidoc/index.asciidoc

@@ -1,4 +1,4 @@
-:curator_version: 8.0.7
+:curator_version: 8.0.8
 :curator_major: 8
 :curator_doc_tree: 8.0
 :es_py_version: 8.8.2

tests/integration/test_integrations.py

@@ -125,9 +125,6 @@ def test_population_check_missing_index(self):
         key = 'docs'
         self.create_index(self.IDX1)
         ilo = IndexList(self.client)
-        assert ilo.indices == [self.IDX1]
-        del ilo.index_info[self.IDX1]
-        assert not ilo.index_info
         ilo.population_check(self.IDX1, key)
         assert ilo.index_info[self.IDX1][key] == 0
     def test_population_check_missing_key(self):

tests/unit/test_class_index_list.py

@@ -92,6 +92,8 @@ def test_empty_list(self):
     def test_get_segmentcount(self):
         self.builder(key='1')
         self.client.indices.segments.return_value = testvars.shards
+        # Ordinarily get_index_state is run before get_segment_counts, so we do so manually here.
+        self.ilo.get_index_state()
         self.ilo.get_segment_counts()
         self.assertEqual(71, self.ilo.index_info[testvars.named_index]['segments'])
 
@@ -106,6 +108,8 @@ def builder(self, key='2'):
         self.ilo = IndexList(self.client)
     def test_get_name_based_ages_match(self):
         self.builder()
+        self.ilo.get_index_state()
+        self.ilo.get_index_settings()
         self.ilo._get_name_based_ages('%Y.%m.%d')
         self.assertEqual(1456963200, self.ilo.index_info['index-2016.03.03']['age']['name'])
     def test_get_name_based_ages_no_match(self):
@@ -256,6 +260,8 @@ def test_creation_date_older_than_now(self):
         self.assertEqual(['index-2016.03.03','index-2016.03.04'], sorted(self.ilo.indices))
     def test_creation_date_older_than_now_raises(self):
         self.builder()
+        self.ilo.get_index_state()
+        self.ilo.get_index_settings()
         self.ilo.index_info['index-2016.03.03']['age'].pop('creation_date')
         self.ilo.index_info['index-2016.03.04']['age'].pop('creation_date')
         self.ilo.filter_by_age(
@@ -270,6 +276,8 @@ def test_creation_date_younger_than_now(self):
         self.assertEqual([], sorted(self.ilo.indices))
     def test_creation_date_younger_than_now_raises(self):
         self.builder()
+        self.ilo.get_index_state()
+        self.ilo.get_index_settings()
         self.ilo.index_info['index-2016.03.03']['age'].pop('creation_date')
         self.ilo.index_info['index-2016.03.04']['age'].pop('creation_date')
         self.ilo.filter_by_age(
@@ -802,6 +810,8 @@ def test_missing_creation_date_raises(self):
         range_to = 0
         expected = []
         self.builder()
+        self.ilo.get_index_state()
+        self.ilo.get_index_settings()
         self.ilo.index_info['index-2016.03.03']['age'].pop('creation_date')
         self.ilo.index_info['index-2016.03.04']['age'].pop('creation_date')
         self.ilo.filter_period(unit=self.unit, range_from=range_from, range_to=range_to,