Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

benchmark/cosbench.py: working version #146

Merged
merged 2 commits into from Apr 23, 2018
Merged

benchmark/cosbench.py: working version #146

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7015cf7
benchmark: create a working version of cosbench
neha-ojha Apr 9, 2018
a29a64a
cosbench.py: add parameter is_teuthology
neha-ojha Apr 18, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
16 changes: 10 additions & 6 deletions benchmark/cosbench.py
View file
Open in desktop
Expand Up @@ -28,27 +28,29 @@ def __init__(self, cluster, config):
self.objects = config["objects_max"]
self.mode = config["mode"]
self.user = settings.cluster.get('user')
self.rgw = settings.cluster.get('rgws')[0]
self.rgw = settings.cluster.get('rgws').keys()[0]
self.radosgw_admin_cmd = settings.cluster.get('radosgw-admin_cmd', '/usr/bin/radosgw-admin')
self.use_existing = settings.cluster.get('use_existing')
self.is_teuthology = settings.cluster.get('is_teuthology', False)

self.run_dir = '%s/osd_ra-%08d/op_size-%s/concurrent_procs-%03d/containers-%05d/objects-%05d/%s' % (self.run_dir, int(self.osd_ra), self.op_size, int(self.total_procs), int(self.containers),int(self.objects), self.mode)
self.out_dir = '%s/osd_ra-%08d/op_size-%s/concurrent_procs-%03d/containers-%05d/objects-%05d/%s' % (self.archive_dir, int(self.osd_ra), self.op_size, int(self.total_procs), int(self.containers),int(self.objects), self.mode)
self.out_dir = self.archive_dir

def prerun_check(self):
#1. check cosbench
if not self.check_workload_status():
sys.exit()
#2. check rgw
cosconf = {}
for param in self.config["auth"]["config"].split(';'):
for param in self.config["auth"].split(';'):
try:
key, value = param.split('=')
cosconf[key] = value
except:
pass
logger.debug("%s", cosconf)
if "username" in cosconf and "password" in cosconf and "url" in cosconf:
if not self.use_existing:
if not self.use_existing or self.is_teuthology:
user, subuser = cosconf["username"].split(':')
stdout, stderr = common.pdsh("%s@%s" % (self.user, self.rgw),"radosgw-admin user create --uid='%s' --display-name='%s'" % (user, user)).communicate()
stdout, stderr = common.pdsh("%s@%s" % (self.user, self.rgw),"radosgw-admin subuser create --uid=%s --subuser=%s --access=full" % (user, cosconf["username"])).communicate()
Expand Down Expand Up @@ -116,7 +118,7 @@ def choose_template(self, temp_name, conf):
"description": conf["mode"],
"name": "%s_%scon_%sobj_%s_%dw" % (conf["mode"], conf["containers_max"], conf["objects_max"], conf["obj_size"], conf["workers"]),
"storage": {"type":"swift", "config":"timeout=300000" },
"auth": {"type":"swauth", "config":"%s" % (conf["auth"]["config"])},
"auth": {"type":"swauth", "config":"%s" % (conf["auth"])},
"workflow": {
"workstage": [{
"name": "main",
Expand Down Expand Up @@ -241,6 +243,7 @@ def run(self):
common.sync_files('%s/archive/%s*' % (self.config["cosbench_dir"], self.runid), self.out_dir)

def check_workload_status(self):
logger.info("Checking workload status")
wait = True
try:
self.runid
Expand All @@ -260,6 +263,7 @@ def check_workload_status(self):
time.sleep(1)
stdout, stderr = common.pdsh("%s@%s" % (self.user, self.config["controller"]),"sh %s/cli.sh info " % (self.config["cosbench_dir"])).communicate()
logger.debug(stdout)
time.sleep(15)
return True

def check_cosbench_res_dir(self):
Expand All @@ -269,7 +273,7 @@ def check_cosbench_res_dir(self):
stdout, stderr = common.pdsh("%s@%s" % (self.user, self.config["controller"]), "find %s/archive -maxdepth 1 -name '%s-*'" % (self.config["cosbench_dir"], self.runid)).communicate()
if stdout:
return True
if check_time == 300:
if check_time == 3000:
return False
check_time += 1
time.sleep(1)
Expand Down
12 changes: 12 additions & 0 deletions cluster/ceph.py
View file
Open in desktop
Expand Up @@ -420,6 +420,18 @@ def start_rgw(self):
pdshhost = '%s@%s' % (user, rgwhost)
common.pdsh(pdshhost, 'sudo sh -c "ulimit -n 16384 && ulimit -c unlimited && exec %s"' % cmd).communicate()

# set min_size of pools to 1, when there is only one osd
num_osds = len(settings.cluster.get('osds'))
rgw_default_pools = ['.rgw.root', 'default.rgw.control', 'default.rgw.meta', 'default.rgw.log']
pool_min_repl_size = 1

if num_osds == 1:
time.sleep(5)
for pool in rgw_default_pools:
common.pdsh(settings.getnodes('head'), 'sudo %s -c %s osd pool set %s min_size %d' % (self.ceph_cmd, self.tmp_conf, pool, pool_min_repl_size),
continue_if_error=False).communicate()
time.sleep(5)


def disable_scrub(self):
common.pdsh(settings.getnodes('head'), "ceph osd set noscrub; ceph osd set nodeep-scrub").communicate()
Expand Down