ceph-gentle-split

#!/usr/bin/python
#
# ceph-gentle-split
# Author: Dan van der Ster <daniel.vanderster@cern.ch>
#
# Slowly split a pool causing minimal impact in a ceph cluster.
#

import sys, getopt, commands, json, time, datetime

def update_osd_dump():
  global osd_dump
  print "update_osd_dump: loading ceph osd dump"
  (status, osd_dump_json) = commands.getstatusoutput('ceph osd dump --format=json 2>/dev/null')
  if status:
    print "Error: %s", osd_dump_json
    sys.exit(1)

  osd_dump = json.loads(osd_dump_json)
  print "update_osd_dump: done"

def get_pg_num(pool_name):
  global osd_dump

  for pool in osd_dump['pools']:
    if pool['pool_name'] == pool_name:
      pg_num = int(pool['pg_num'])
      print "get_pg_num: %s has pg_num %s" % (pool_name, pg_num)
      return pg_num
  raise Exception('Undefined pg_num for %s' % pool_name)

def get_pgp_num(pool_name):
  global osd_dump

  for pool in osd_dump['pools']:
    if pool['pool_name'] == pool_name:
      pgp_num = int(pool['pg_placement_num'])
      print "get_pgp_num: %s has pgp_num %s" % (pool_name, pgp_num)
      return pgp_num
  raise Exception('Undefined pgp_num for %s' % pool_name)

def in_timeframe(start_time, end_time, current_time, allowed_days, current_day):
  print "check current day: %s" % (current_day)

  if current_day not in allowed_days:
    print '%s not in allowed_days %s' % (current_day, allowed_days)
    return False

  print current_time.strftime('check current time: %H:%M:%S')
  if start_time < end_time:
    if current_time >= start_time and current_time <= end_time:
      time_ok = True
    else:
      time_ok = False
  elif current_time >= start_time or current_time <= end_time: #Over Midnight
    time_ok = True
  else:
    time_ok = False

  return time_ok

def measure_latency(test_pool):
  print "measure_latency: measuring 4kB write latency"
  cmd="rados -p %s bench 10 write -t 1 -b 4096 2>/dev/null | egrep -i 'average latency' | awk '{print $3}'" % (test_pool)
  latency = commands.getoutput(cmd)
  latency_ms = 1000*float(latency)
  print "measure_latency: current latency is %s" % latency_ms
  return latency_ms

def get_num_inactive():
  cmd = "ceph pg ls |grep '^[0-9]'| egrep -v 'PG_STAT|active' | wc -l"
  out = commands.getoutput(cmd)
  n = int(out)
  print "get_num_inactive: PGs currently inactive: %s" % n
  return n

def get_num_backfilling():
  cmd = "ceph pg ls backfilling backfill_wait | grep -v PG_STAT | wc -l"
  out = commands.getoutput(cmd)
  n = int(out)
  print "get_num_backfilling: PGs currently backfilling: %s" % n
  return n

def split_pool(pool, pg_num, really):
  cmd = "ceph osd pool set %s pg_num %s" % (pool, pg_num)
  print "split_pool: calling %s" % cmd
  if really:
    (status, out) = commands.getstatusoutput(cmd)
    if status:
      print "Error: %s", out
      sys.exit(1)

    print "split_pool: %s" % out
    time.sleep(5)

    # wait until those pgs are created
    while get_num_inactive():
      time.sleep(5)

    cmd = "ceph osd pool set %s pgp_num %s" % (pool, pg_num)
    status = 1
    while status != 0:
      print "split_pool: calling %s" % cmd
      (status, out) = commands.getstatusoutput(cmd)
      print "split_pool: %s" % out
      time.sleep(2)
    return
  print "split_pool: not really doing it!"

def split(pool, test_pool, goal, max_pgs_backfilling, max_latency, max_increment, start_time, end_time, allowed_days, really):

  # check if there is any work to do:
  update_osd_dump()
  current_pg_num = get_pg_num(pool)
  current_pgp_num = get_pgp_num(pool)

  if current_pg_num != current_pgp_num:
    print "split: pg_num != pgp_num. fix that then start again"
    sys.exit(1)

  if current_pg_num >= goal:
    print "split: no work to do, exiting"
    sys.exit(0)

  print "split: splitting from %s to %s" % (current_pg_num, goal)

  # check timeframe
  current_time = datetime.datetime.now().time()
  current_day = datetime.datetime.now().weekday()
  time_ok = in_timeframe(start_time, end_time, current_time, allowed_days, current_day)
  if not time_ok:
    print 'current datetime not within valid range, trying again later'
    return

  if max_pgs_backfilling > 0:
    # check num pgs backfilling
    npgs = get_num_backfilling()
    if npgs >= max_pgs_backfilling:
      print "split: npgs backfilling is too high, trying again later"
      return

  if max_latency > 0:
    # check the latency
    latency = measure_latency(test_pool)
    if latency > max_latency:
      print "split: latency is too high, trying again later"
      return

  new_pg_num = min(goal, current_pg_num + max_increment)
  print "split: %s new pg_num will be %s" % (pool, new_pg_num)
  split_pool(pool, new_pg_num, really)

  if new_pg_num == goal:
    print "All done"
    sys.exit(0)

def usage(code=0):
  print 'ceph-gentle-split -p <pool> [-t <latency-test-pool> (default=test)] -g <goal num pgs> [-l <max_latency (default=15)>] [-b <max pgs backfilling (default=10)>] [-s <start time (default=02:00)>] [-e <end time (default=09:00)>] [-a <day of week,[day of week,...] [-i <max increment (default=10)>]'
  sys.exit(code)

def main(argv):
  pool = ''
  test_pool = 'test'
  max_latency = 50
  max_pgs_backfilling = 20
  max_increment = 10
  goal = 0
  start_time = "02:00"
  end_time = "09:00"
  allowed_days = [0,1,2,3,4,5,6]
  really = False
  delay = 60

  try:
    opts, args = getopt.getopt(argv,"hp:t:l:b:s:g:i:a:e:r:d",["pool=","test_pool=","latency=","backfills=","increment=","goal=","allowed_days=","start_time=","end_time=","really","delay="])
  except getopt.GetoptError:
    usage(2)
  for opt, arg in opts:
    if opt == '-h':
      usage()
    elif opt in ("-p", "--pool"):
      pool = arg
    elif opt in ("-t", "--test-pool"):
      test_pool = arg
    elif opt in ("-l", "--latency"):
      max_latency = int(arg)
    elif opt in ("-b", "--backfills"):
      max_pgs_backfilling = int(arg)
    elif opt in ("-i", "--increment"):
      max_increment = int(arg)
    elif opt in ("-g", "--goal"):
      goal = int(arg)
    elif opt in ("-s", "--start-time"):
      start_time = str(arg)
    elif opt in ("-e", "--end-time"):
      end_time = str(arg)
    elif opt in ("-a", "--allowed-days"):
      allowed_days = arg.split(',')
    elif opt in ("-r", "--really"):
      really = True
    elif opt in ("-d", "--delay"):
      delay = int(arg)
  if not pool or goal < 1:
    usage(2)

  end_time = datetime.datetime.strptime(end_time, "%H:%M").time()
  start_time = datetime.datetime.strptime(start_time, "%H:%M").time()
  if allowed_days:
    allowed_days = map(int, allowed_days)

  print 'Pool:', pool
  print 'Latency test pool:', test_pool
  print 'Goal: ', goal
  print 'Max latency (ms):', max_latency
  print 'Max PGs backfilling:', max_pgs_backfilling
  print 'Max increment:', max_increment
  print 'Start time:', start_time
  print 'End time:', end_time
  print 'Allowed days:', allowed_days
  print 'Delay:', delay

  while(True):
    split(pool, test_pool, goal, max_pgs_backfilling, max_latency, max_increment, start_time, end_time, allowed_days, really)
    print "main: sleeping %s seconds" % delay
    time.sleep(delay)

if __name__ == "__main__":
  main(sys.argv[1:])