Skip to content

Commit

Permalink
Merge pull request #1290 from tschroeder-zendesk/sort-aged-indices-by…
Browse files Browse the repository at this point in the history 
…-alpha

Sort index list alphabetically prior to sorting by age
  • Loading branch information
@untergeek
untergeek committed Oct 30, 2018
2 parents 0a650c6 + eab847f commit b66fb65
Show file tree
Hide file tree
Showing 5 changed files with 90 additions and 21 deletions.
9 changes: 6 additions & 3 deletions curator/indexlist.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -255,7 +255,7 @@ def _get_segment_counts(self):
if err.status_code == 413:
self.loggit.debug('Huge Payload 413 Error - Trying to get information with multiple requests')
working_list = {}
working_list.update(self._bulk_queries(l, self._get_indices_segments))
working_list.update(self._bulk_queries(l, self._get_indices_segments))

if working_list:
for index in list(working_list.keys()):
Expand Down Expand Up @@ -397,14 +397,17 @@ def _sort_by_age(self, index_list, reverse=True):
' metadata'.format(index, self.age_keyfield)
)
self.__excludify(True, True, index, msg)

# Sort alphabetically prior to age sort to keep sorting consistent
temp_tuple = (
sorted(temp.items(), key=lambda k: k[0], reverse=reverse)
)
# If reverse is True, this will sort so the youngest indices are first.
# However, if you want oldest first, set reverse to False.
# Effectively, this should set us up to act on everything older than
# meets the other set criteria.
# It starts as a tuple, but then becomes a list.
sorted_tuple = (
sorted(temp.items(), key=lambda k: k[1], reverse=reverse)
sorted(temp_tuple, key=lambda k: k[1], reverse=reverse)
)
return [x[0] for x in sorted_tuple]

Expand Down
10 changes: 5 additions & 5 deletions curator/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ def single_constructor(loader,node):
return os.environ[envvar] if envvar in os.environ else default

yaml.add_constructor('!single', single_constructor)

try:
return yaml.load(read_file(path))
except yaml.scanner.ScannerError as err:
Expand Down Expand Up @@ -1841,7 +1841,7 @@ def get_datemath(client, datemath, random_element=None):
"""
if random_element is None:
randomPrefix = (
'curator_get_datemath_function_' +
'curator_get_datemath_function_' +
''.join(random.choice(string.ascii_lowercase) for _ in range(32))
)
else:
Expand Down Expand Up @@ -1884,8 +1884,8 @@ def isdatemath(data):

def parse_datemath(client, value):
"""
Check if ``value`` is datemath.
Parse it if it is.
Check if ``value`` is datemath.
Parse it if it is.
Return the bare value otherwise.
"""
if not isdatemath(value):
Expand All @@ -1906,4 +1906,4 @@ def parse_datemath(client, value):
suffix = r.match(value).group(4) or ''
except AttributeError:
raise exceptions.ConfigurationError('Value "{0}" does not contain a valid datemath pattern.'.format(value))
return '{0}{1}{2}'.format(prefix, get_datemath(client, datemath), suffix)
return '{0}{1}{2}'.format(prefix, get_datemath(client, datemath), suffix)
4 changes: 2 additions & 2 deletions test/integration/test_allocation.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ def test_remove_exclude_with_none_value(self):
self.create_index('not_my_index')
# Put a setting in place before we start the test.
self.client.indices.put_settings(
index='my_index',
index='my_index',
body={'index.routing.allocation.{0}.{1}'.format(at, key): 'bar'}
)
# Ensure we _have_ it here first.
Expand Down Expand Up @@ -216,4 +216,4 @@ def test_include(self):
self.assertEquals(value,
self.client.indices.get_settings(index='my_index')['my_index']['settings']['index']['routing']['allocation'][at][key])
self.assertNotIn('routing',
self.client.indices.get_settings(index='not_my_index')['not_my_index']['settings']['index'])
self.client.indices.get_settings(index='not_my_index')['not_my_index']['settings']['index'])
63 changes: 53 additions & 10 deletions test/integration/test_count_pattern.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,6 @@

host, port = os.environ.get('TEST_ES_SERVER', 'localhost:9200').split(':')
port = int(port) if port else 9200
# ' - filtertype: {0}\n'
# ' source: {1}\n'
# ' direction: {2}\n'
# ' timestring: {3}\n'
# ' unit: {4}\n'
# ' unit_count: {5}\n'
# ' field: {6}\n'
# ' stats_result: {7}\n'
# ' epoch: {8}\n')

global_client = elasticsearch.Elasticsearch(host=host, port=port)

Expand Down Expand Up @@ -96,4 +87,56 @@ def test_match_proper_indices_by_age(self):
],
)
indices = sorted(list(self.client.indices.get('_all')))
self.assertEquals(['a-2017.10.03', 'b-2017.09.03', 'not_a_match'], indices)
self.assertEquals(['a-2017.10.03', 'b-2017.09.03', 'not_a_match'], indices)
def test_count_indices_by_age_same_age(self):
key = 'tag'
value = 'value'
at = 'include'
self.write_config(
self.args['configfile'], testvars.client_config.format(host, port))
self.write_config(self.args['actionfile'],
testvars.allocation_count_test.format(key, value, at, False))
self.create_index('c-2017.10.01')
self.create_index('c-2017.10.02')
self.create_index('c-2017.10.03')
self.create_index('a-2017.10.01')
self.create_index('a-2017.10.02')
self.create_index('a-2017.10.03')
self.create_index('b-2017.10.01')
self.create_index('b-2017.10.02')
self.create_index('b-2017.10.03')
self.create_index('d-2017.10.01')
self.create_index('d-2017.10.02')
self.create_index('d-2017.10.03')
test = clicktest.CliRunner()
result = test.invoke(
curator.cli,
[
'--config', self.args['configfile'],
self.args['actionfile']
],
)
self.assertEquals(value,
self.client.indices.get_settings(index='c-2017.10.03')['c-2017.10.03']['settings']['index']['routing']['allocation'][at][key])
self.assertEquals(value,
self.client.indices.get_settings(index='d-2017.10.03')['d-2017.10.03']['settings']['index']['routing']['allocation'][at][key])
self.assertNotIn('routing',
self.client.indices.get_settings(index='a-2017.10.01')['a-2017.10.01']['settings']['index'])
self.assertNotIn('routing',
self.client.indices.get_settings(index='a-2017.10.02')['a-2017.10.02']['settings']['index'])
self.assertNotIn('routing',
self.client.indices.get_settings(index='a-2017.10.03')['a-2017.10.03']['settings']['index'])
self.assertNotIn('routing',
self.client.indices.get_settings(index='b-2017.10.01')['b-2017.10.01']['settings']['index'])
self.assertNotIn('routing',
self.client.indices.get_settings(index='b-2017.10.02')['b-2017.10.02']['settings']['index'])
self.assertNotIn('routing',
self.client.indices.get_settings(index='b-2017.10.03')['b-2017.10.03']['settings']['index'])
self.assertNotIn('routing',
self.client.indices.get_settings(index='c-2017.10.01')['c-2017.10.01']['settings']['index'])
self.assertNotIn('routing',
self.client.indices.get_settings(index='c-2017.10.02')['c-2017.10.02']['settings']['index'])
self.assertNotIn('routing',
self.client.indices.get_settings(index='d-2017.10.01')['d-2017.10.01']['settings']['index'])
self.assertNotIn('routing',
self.client.indices.get_settings(index='d-2017.10.02')['d-2017.10.02']['settings']['index'])
25 changes: 24 additions & 1 deletion test/integration/testvars.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,29 @@
' kind: prefix\n'
' value: my\n')


allocation_count_test = ('---\n'
'actions:\n'
' 1:\n'
' description: "Allocate by key/value/allocation_type"\n'
' action: allocation\n'
' options:\n'
' key: {0}\n'
' value: {1}\n'
' allocation_type: {2}\n'
' wait_for_completion: {3}\n'
' wait_interval: 1\n'
' max_wait: -1\n'
' continue_if_exception: False\n'
' disable_action: False\n'
' filters:\n'
' - filtertype: count\n'
' use_age: True\n'
' source: name\n'
' timestring: \'%Y.%m.%d\'\n'
' count: 2\n'
' exclude: False\n')

cluster_routing_test = ('---\n'
'actions:\n'
' 1:\n'
Expand Down Expand Up @@ -846,4 +869,4 @@
' unit_count: {5}\n'
' field: {6}\n'
' stats_result: {7}\n'
' epoch: {8}\n')
' epoch: {8}\n')

0 comments on commit b66fb65

Please sign in to comment.