Permalink
Browse files

improve curator:

- fix broken name filtering because existing dates in indexes create issues
- allow archive box realocation to be disabled via LOGS_DB_HAS_NO_ARCHIVE
  • Loading branch information...
Schnitzel committed Dec 1, 2018
1 parent 68002f9 commit 7949a033a45798c947efe478f1d95cebe43322e2
Showing with 36 additions and 34 deletions.
  1. +1 −3 services/logs-db-curator/Dockerfile
  2. +34 −30 services/logs-db-curator/actions.yml
  3. +1 −1 services/logs-db-curator/curator.yml
@@ -4,7 +4,5 @@ FROM ${IMAGE_REPO:-lagoon}/curator
COPY actions.yml curator.yml /curator/
COPY 90-generate-http-auth.sh /lagoon/entrypoints/

RUN echo "source /lagoon/entrypoints/90-generate-http-auth.sh" >> /home/.bashrc

ENV LAGOON_INDEXES="^(container-logs-|router-logs-|service-logs-|application-logs-|lagoon-logs-).*$"
ENV CRONJOBS="01 0 * * * /usr/bin/curator --config curator.yml actions.yml"
ENV CRONJOBS="01 0 * * * /usr/bin/curator --config /curator/curator.yml /curator/actions.yml"
@@ -2,8 +2,7 @@ actions:
1:
action: index_settings
description: >-
Set lagoon indices older than 40 days to be read only (block writes)
and to use best_compression codec
Set lagoon indices not created in current month to be read only (block writes)
options:
ignore_empty_list: True
index_settings:
@@ -14,17 +13,17 @@ actions:
- filtertype: pattern
kind: regex
value: ${LAGOON_INDEXES}
- filtertype: age
source: name
timestring: '%Y.%m'
direction: older
unit: ${OLDER_UNIT:days}
unit_count: ${OLDER_UNIT_COUNT:40}
- filtertype: period
period_type: relative
source: creation_date
range_from: 0
range_to: 1
unit: months
exclude: True
2:
action: replicas
description: >-
Reduce the replica count to 0 for lagoon indices older than
40 days
Reduce the replica count to 0 for lagoon indices not created in current month
options:
ignore_empty_list: True
count: 0
@@ -33,33 +32,37 @@ actions:
- filtertype: pattern
kind: regex
value: ${LAGOON_INDEXES}
- filtertype: age
source: name
timestring: '%Y.%m'
direction: older
unit: ${OLDER_UNIT:days}
unit_count: ${OLDER_UNIT_COUNT:40}
- filtertype: period
period_type: relative
source: creation_date
range_from: 0
range_to: 1
unit: months
exclude: True
3:
action: allocation
description: >-
Apply shard allocation routing to 'require' 'box_type=archive' for hot/cold node
setup for lagoon indices older than 40 days
setup for lagoon indices not created in current month
options:
ignore_empty_list: True
key: box_type
value: archive
allocation_type: require
wait_for_completion: True
max_wait: 1800
disable_action: ${LOGS_DB_HAS_NO_ARCHIVE:False}
filters:
- filtertype: pattern
kind: regex
value: ${LAGOON_INDEXES}
- filtertype: age
source: name
timestring: '%Y.%m'
direction: older
unit: ${OLDER_UNIT:days}
unit_count: ${OLDER_UNIT_COUNT:40}
- filtertype: period
period_type: relative
source: creation_date
range_from: 0
range_to: 1
unit: months
exclude: True
- filtertype: allocated
key: box_type
value: archive
@@ -68,7 +71,7 @@ actions:
4:
action: forcemerge
description: >-
forceMerge lagoon indices older than 40 days to 1 segments per shard.
forceMerge lagoon indices not created in current month to 1 segments per shard.
Delay 20 seconds between each forceMerge operation to allow the cluster
to quiesce. Skip indices that have already been forcemerged to the
minimum number of segments to avoid reprocessing.
@@ -81,12 +84,13 @@ actions:
- filtertype: pattern
kind: regex
value: ${LAGOON_INDEXES}
- filtertype: age
source: name
timestring: '%Y.%m'
direction: older
unit: ${OLDER_UNIT:days}
unit_count: ${OLDER_UNIT_COUNT:40}
- filtertype: period
period_type: relative
source: creation_date
range_from: 0
range_to: 1
unit: months
exclude: True
- filtertype: forcemerged
max_num_segments: 1
exclude: True
@@ -3,7 +3,7 @@ client:
hosts:
- logs-db
port: 9200
http_auth: ${LOGSDB_ADMIN_PASSWORD}
http_auth: ${LOGSDB_HTTP_AUTH}

logging:
loglevel: INFO

0 comments on commit 7949a03

Please sign in to comment.