/
force-unfreeze.py
31 lines (23 loc) · 1.28 KB
/
force-unfreeze.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
"""Unfreeze writes forcefully in case they are stuck to frozen."""
import argparse
import logging
from cookbooks.sre.elasticsearch import CLUSTERGROUPS
__title__ = __doc__
logger = logging.getLogger(__name__)
def argument_parser():
"""As specified by Spicerack API."""
parser = argparse.ArgumentParser(prog=__name__, description=__title__,
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('clustergroup', choices=CLUSTERGROUPS, help='Name of clustergroup. One of: %(choices)s.')
parser.add_argument('admin_reason', help='Administrative Reason')
parser.add_argument('--task-id', help='task_id for the change')
return parser
def run(args, spicerack):
"""Required by Spicerack API."""
elasticsearch_clusters = spicerack.elasticsearch_clusters(args.clustergroup, args.write_queue_datacenters)
reason = spicerack.admin_reason(args.admin_reason, task_id=args.task_id)
# If frozen state is in an unstable state, freezing and thawing writes can
# restore a normal situation. See incident report below for details.
# https://wikitech.wikimedia.org/wiki/Incident_documentation/20190327-elasticsearch
with elasticsearch_clusters.frozen_writes(reason):
logger.info('Thawing writes again!')