From 5c9520b7c809684ca993173e3300eb22bd2fe625 Mon Sep 17 00:00:00 2001 From: Somnath Date: Thu, 18 Mar 2021 10:36:16 +0530 Subject: [PATCH] m0crate kv(index) test fixes. (#461) * Removed unwanted info from kv test run o/p, added KEY_SIZE support. * Added m0crate index value_size changes. * Cleaned record size changes and added changes for max key and value. * Added changes for total op time and time per ops in ns. * Removed RECORD_SIZE and MAX_RSIZE from tests/*.yaml files. Signed-off-by: somnathbghule --- motr/m0crate/crate.c | 16 +- motr/m0crate/crate_client.h | 9 +- motr/m0crate/crate_index.c | 252 ++++++++++++++---- motr/m0crate/parser.c | 62 +++-- motr/m0crate/scripts/README.md | 61 +++++ .../scripts/gen_index_yaml_run_workload | 110 ++++++++ .../scripts/m0crate-index.yaml.template | 59 ++++ motr/m0crate/tests/test1.yaml | 69 ++--- motr/m0crate/tests/test2.yaml | 69 ++--- motr/m0crate/tests/test3.yaml | 69 ++--- motr/m0crate/tests/test4.yaml | 69 ++--- motr/m0crate/tests/test5.yaml | 69 ++--- motr/m0crate/tests/test6.yaml | 69 ++--- 13 files changed, 708 insertions(+), 275 deletions(-) create mode 100644 motr/m0crate/scripts/README.md create mode 100755 motr/m0crate/scripts/gen_index_yaml_run_workload create mode 100644 motr/m0crate/scripts/m0crate-index.yaml.template diff --git a/motr/m0crate/crate.c b/motr/m0crate/crate.c index 266b12c0067..42242486ec4 100644 --- a/motr/m0crate/crate.c +++ b/motr/m0crate/crate.c @@ -65,6 +65,9 @@ const int cr_default_nr_thread = 1; const short cr_default_read_frac = 50; const bcnt_t cr_default_blocksize = 16 * 1024; const bcnt_t cr_default_csum_size = 16; +const bcnt_t cr_default_key_size = sizeof(struct m0_fid); +const bcnt_t cr_default_max_ksize = 1 << 10; /* default upper limit for key_size parameter. i.e 1KB */ +const bcnt_t cr_default_max_vsize = 1 << 20; /* default upper limit for value_size parameter. i.e 1MB */ const char *cr_workload_name[CWT_NR] = { @@ -224,6 +227,16 @@ int workload_init(struct workload *w, enum cr_workload_type wtype) w->cw_nr_dir = cr_default_nr_dir; w->cw_read_frac = cr_default_read_frac; + /* set default values before yaml parsing */ + if (wtype == CWT_INDEX) { + struct m0_workload_index *wit = w->u.cw_index; + wit->key_size = cr_default_key_size; + /* default value_size is -1 i.e random */ + wit->value_size = -1; + wit->max_key_size = cr_default_max_ksize; + wit->max_value_size = cr_default_max_vsize; + } + return wop(w)->wto_init(w); } @@ -380,7 +393,8 @@ static void workload_run(struct workload *w) w->cw_name, w->cw_type); cr_log(CLL_INFO, "random seed: %u\n", w->cw_rstate); cr_log(CLL_INFO, "number of threads: %u\n", w->cw_nr_thread); - if (CWT_IO != w->cw_type) { + /* Following params not applicable to IO and INDEX tests */ + if (CWT_IO != w->cw_type && CWT_INDEX != w->cw_type) { cr_log(CLL_INFO, "average size: %llu\n", w->cw_avg); cr_log(CLL_INFO, "maximal size: %llu\n", w->cw_max); /* diff --git a/motr/m0crate/crate_client.h b/motr/m0crate/crate_client.h index cdd3b063123..f2b407977e7 100644 --- a/motr/m0crate/crate_client.h +++ b/motr/m0crate/crate_client.h @@ -80,7 +80,6 @@ struct m0_workload_index { int num_kvs; int mode; int opcode; - int record_size; int opcode_prcnt[CRATE_OP_TYPES]; int next_records; @@ -102,11 +101,17 @@ struct m0_workload_index { struct m0_fid key_prefix; int keys_count; + /** Added to set key_size and value_size parameter from .yaml file. */ + int key_size; + int value_size; + int max_key_size; + int max_value_size; + int min_key_size; + bool keys_ordered; struct m0_fid index_fid; - int max_record_size; uint64_t seed; }; diff --git a/motr/m0crate/crate_index.c b/motr/m0crate/crate_index.c index 360b862f377..7cdabcf7848 100644 --- a/motr/m0crate/crate_index.c +++ b/motr/m0crate/crate_index.c @@ -72,11 +72,12 @@ * * WORKLOAD_TYPE - always 0 (idx operations). * * WORKLOAD_SEED - initial value for pseudo-random generator * (int or "tstamp"). - * * RECORD_SIZE - size of record; can be defined as number, number with - * units (1M) or "random". Value should be > 16 because - * `RECORD_SIZE = sizeof(struct m0_fid) + value_len`. - * * MAX_RSIZE - max size of record; used in case, when RECORD_SIZE="random". - * Also, see ::CR_MAX_OF_MAX_RECORD_SIZE. + * * KEY_SIZE - size of the key; can be defined as number or "random", KEY_SIZE should be >= 16 + * * VALUE_SIZE - size of the value; can be defined as number or "random" + * * MAX_KEY_SIZE - max size of key; used in case, when KEY_SIZE = "random". + * Also, see ::CR_MAX_OF_MAX_KEY_SIZE. + * * MAX_VALUE_SIZE - max size of value; used in case, when VALUE_SIZE = "random". + * Also, see ::CR_MAX_OF_MAX_VALUE_SIZE. * * OP_COUNT - total operations count; can be defined as number, number with * units (1K), or "unlimited". * * PUT,GET,DEL,NEXT - percents of portion operations. @@ -158,10 +159,14 @@ /** Global constants for DIX workload */ enum { - /** Upper limit for max_record_size parameter. */ - CR_MAX_OF_MAX_RECORD_SIZE = (1 << 30), - /** Upper limit for next_records parameter. */ - CR_MAX_NEXT_RECORDS = (1 << 30), + /** Upper limit for max_key_size parameter. i.e 1MB */ + CR_MAX_OF_MAX_KEY_SIZE = (1 << 20), + /** Lower limit for key_size parameter. i.e 16 Bytes */ + CR_MIN_KEY_SIZE = sizeof(struct m0_fid), + /** Upper limit for max_value_size parameter. i.e 1GB */ + CR_MAX_OF_MAX_VALUE_SIZE = (1 << 30), + /** Upper limit for next_records parameter. ie 1GB */ + CR_MAX_NEXT_RECORDS = (1 << 30), }; enum cr_op_selector { @@ -229,6 +234,45 @@ struct cr_time_measure_ctx { double elapsed; }; +/* CLIENT INDEX RESULT MEASUREMENTS */ +char *cr_idx_op_labels[CRATE_OP_NR] = { + "PUT", + "GET", + "NEXT", + "DEL" +}; +struct cr_idx_ops_result { + /** op label */ + const char *cior_op_label; + /** comes from cr_opcode::CRATE_OP_PUT */ + int cior_op_type; + /** total ops count per op type */ + int cior_op_count; + /** total op time in m0_time_t */ + m0_time_t cior_ops_total_time_m0; + /** total op time in seconds */ + double cior_ops_total_time_s; + /** per op time in nanoseconds */ + double cior_time_per_op_ns; +}; + +struct cr_idx_w_results { + /** entire workload time in m0 */ + m0_time_t ciwr_total_time_m0; + /** entire workload time in seconds */ + double ciwr_total_time_s; + /** entire workload per op time in ns */ + double ciwr_time_per_op_ns; + /** per op result for each put, get, next and del */ + struct cr_idx_ops_result ciwr_ops_result[CRATE_OP_NR]; +}; + +static double cr_time_in_seconds(m0_time_t mtime) +{ + return (m0_time_seconds(mtime)) + + ( m0_time_nanoseconds(mtime) / (double) M0_TIME_ONE_SECOND ); +} + static void cr_time_measure_begin(struct cr_time_measure_ctx *t) { *t = (struct cr_time_measure_ctx) {}; @@ -245,12 +289,7 @@ static double cr_time_measure_elapsed_now(struct cr_time_measure_ctx *t) static void cr_time_measure_end(struct cr_time_measure_ctx *t) { t->elapsed = cr_time_measure_elapsed_now(t); -} - -static void cr_time_measure_report(struct cr_time_measure_ctx *t, - const char *op) -{ - fprintf(stdout, "result: %s, %f\n", op, t->elapsed); + t->ts_next = m0_time_now(); } /* BITMAP */ @@ -308,6 +347,7 @@ struct cr_idx_w { struct cr_time_measure_ctx exec_time_ctx; size_t exec_time; enum cr_op_selector op_selector; + struct cr_idx_w_results ciw_results; }; static int cr_idx_w_init(struct cr_idx_w *ciw, @@ -343,6 +383,70 @@ static bool cr_idx_w_rebalance_ops(struct cr_idx_w *w, enum cr_opcode op); static int cr_idx_w_common(struct cr_idx_w *w); static int cr_idx_w_warmup(struct cr_idx_w *w); +/** capture final results */ +static void cr_time_capture_results(struct cr_time_measure_ctx *t, + struct cr_idx_w *ciw) +{ + int i = 0; + struct cr_idx_ops_result *op_results = ciw->ciw_results.ciwr_ops_result; + ciw->ciw_results.ciwr_total_time_s = t->elapsed; + ciw->ciw_results.ciwr_total_time_m0 = m0_time_sub(t->ts_next, t->ts); + ciw->ciw_results.ciwr_time_per_op_ns = m0_time_nanoseconds( + ciw->ciw_results.ciwr_total_time_m0 / ciw->nr_ops_total); + + /* Calculate Results for each op (PUT, GET, NEXT and DEL) */ + for (i = 0; i < CRATE_OP_NR; i++) { + if (op_results[i].cior_op_count) { + op_results[i].cior_ops_total_time_s = + cr_time_in_seconds(op_results[i].cior_ops_total_time_m0); + op_results[i].cior_time_per_op_ns = m0_time_nanoseconds( + op_results[i].cior_ops_total_time_m0 / + op_results[i].cior_op_count); + } + } + +} +/** RESULT REPORT */ +static void cr_time_measure_report(struct cr_time_measure_ctx *t, + struct cr_idx_w w) +{ + int i = 0; + struct cr_idx_ops_result *op_results = w.ciw_results.ciwr_ops_result; + + /* Results in parsable format */ + fprintf(stdout, "result: total_s, %f, avg_time_per_op_ns, %.1f," + " key_size_bytes, %d, value_size_bytes, %d, ops, %d\n", + t->elapsed, w.ciw_results.ciwr_time_per_op_ns, + w.wit->key_size, w.wit->value_size, w.nr_ops_total); + + for (i = 0; i < CRATE_OP_NR; i++) { + fprintf(stdout, "result: %s, total_time_s, %f, avg_time_per_op_ns, " + "%.1f, ops, %d\n", + op_results[i].cior_op_label, + op_results[i].cior_ops_total_time_s, + op_results[i].cior_time_per_op_ns, + op_results[i].cior_op_count); + } + + /* Results in m0crate format */ + fprintf(stdout, "\nTotal: time="TIME_F" ("TIME_F" per op) ops=%d\n", + TIME_P(w.ciw_results.ciwr_total_time_m0), + TIME_P(w.ciw_results.ciwr_total_time_m0 / + w.nr_ops_total), + w.nr_ops_total); + + for (i = 0; i < CRATE_OP_NR; i++) { + if (op_results[i].cior_op_count) { + fprintf(stdout, "%s: "TIME_F" ("TIME_F" per op) ops=%d\n", + op_results[i].cior_op_label, + TIME_P(op_results[i].cior_ops_total_time_m0), + TIME_P(op_results[i].cior_ops_total_time_m0 / + op_results[i].cior_op_count), + op_results[i].cior_op_count); + } + } +} + /* IMPLEMENTATION */ @@ -352,7 +456,8 @@ static int cr_idx_w_init(struct cr_idx_w *ciw, int rc; int i; - M0_PRE(wit->max_record_size < CR_MAX_OF_MAX_RECORD_SIZE); + M0_PRE(wit->max_key_size < CR_MAX_OF_MAX_KEY_SIZE); + M0_PRE(wit->max_value_size < CR_MAX_OF_MAX_VALUE_SIZE); M0_PRE(wit->next_records < CR_MAX_NEXT_RECORDS); *ciw = (typeof(*ciw)) {}; @@ -360,6 +465,19 @@ static int cr_idx_w_init(struct cr_idx_w *ciw, ciw->wit = wit; ciw->nr_ops_total = 0; + /* Setting up min_key_size default to MIN_KEY_SIZE */ + ciw->wit->min_key_size = CR_MIN_KEY_SIZE; + + /* Init crate index result */ + for (i = 0; i < CRATE_OP_NR; i++) { + ciw->ciw_results.ciwr_ops_result[i].cior_op_label = + cr_idx_op_labels[i]; + ciw->ciw_results.ciwr_ops_result[i].cior_op_type = i; + ciw->ciw_results.ciwr_ops_result[i].cior_ops_total_time_s = 0; + ciw->ciw_results.ciwr_ops_result[i].cior_time_per_op_ns = 0; + ciw->ciw_results.ciwr_ops_result[i].cior_ops_total_time_m0 = 0; + } + /* XXX: If opcount is unlimited, then make it limited */ if (wit->op_count < 0) { wit->op_count = INT_MAX / (128 * wit->num_kvs); @@ -369,6 +487,8 @@ static int cr_idx_w_init(struct cr_idx_w *ciw, ciw->nr_ops[i].nr = wit->op_count * ((double) wit->opcode_prcnt[i] / 100); ciw->nr_ops_total += ciw->nr_ops[i].nr; + ciw->ciw_results.ciwr_ops_result[i].cior_op_count = + ciw->nr_ops[i].nr; } if (wit->warmup_put_cnt == -1) { ciw->warmup_put_cnt = ciw->nr_ops_total; @@ -383,6 +503,11 @@ static int cr_idx_w_init(struct cr_idx_w *ciw, ciw->nr_keys *= ciw->nr_kv_per_op; ciw->ordered_keys = wit->keys_ordered; + /* If key size is random, generate it using rand(). */ + if (ciw->wit->key_size < 0) + ciw->wit->key_size = ciw->wit->min_key_size + + cr_rand_pos_range(ciw->wit->max_key_size - ciw->wit->min_key_size); + if (wit->warmup_del_ratio > 0) ciw->warmup_del_cnt = ciw->warmup_put_cnt / wit->warmup_del_ratio; @@ -408,12 +533,14 @@ static int cr_idx_w_init(struct cr_idx_w *ciw, else ciw->op_selector = CR_OP_SEL_RR; - /* XXX: if DEL and PUT balanced, then we have to + /* if DEL or GET or NEXT and PUT given, then we have to * use round-robin selector to avoid cases when * we perform GET/DEL/NEXT on empty index. */ - if (ciw->nr_ops[CRATE_OP_DEL].nr == ciw->nr_ops[CRATE_OP_PUT].nr) + if ((ciw->nr_ops[CRATE_OP_DEL].nr != 0 || ciw->nr_ops[CRATE_OP_GET].nr != 0 || + ciw->nr_ops[CRATE_OP_NEXT].nr != 0) && ciw->nr_ops[CRATE_OP_PUT].nr != 0) { ciw->op_selector = CR_OP_SEL_RR; + } M0_POST(ciw->nr_kv_per_op < ciw->nr_keys); M0_POST(ciw->nr_kv_per_op > 0); @@ -527,14 +654,14 @@ static struct m0_bufvec* idx_bufvec_alloc(int nr) static size_t cr_idx_w_get_value_size(struct cr_idx_w *w) { int vlen; - - if (w->wit->record_size < 0) - vlen = sizeof(struct m0_fid) + - cr_rand_pos_range(w->wit->max_record_size); + /* If value_size is random, generate it using rand() */ + if (w->wit->value_size < 0) + vlen = w->wit->key_size + + cr_rand_pos_range(w->wit->max_value_size - w->wit->key_size); else - vlen = w->wit->record_size - sizeof(struct m0_fid); + vlen = w->wit->value_size; - M0_POST(vlen < CR_MAX_OF_MAX_RECORD_SIZE); + M0_POST(vlen < CR_MAX_OF_MAX_VALUE_SIZE); M0_POST(vlen > 0); return vlen; } @@ -553,7 +680,8 @@ static int cr_idx_w_get_nr_keys_per_op(struct cr_idx_w *w, enum cr_opcode op) } static int fill_kv_del(struct cr_idx_w *w, - struct m0_fid *k, struct kv_pair *p, size_t nr) + struct m0_fid *k, struct kv_pair *p, size_t nr, + int kpart_one_size, char *kpart_one) { int i; @@ -566,18 +694,19 @@ static int fill_kv_del(struct cr_idx_w *w, M0_ASSERT(p->k->ov_vec.v_nr == nr); for (i = 0; i < nr; i++) { - p->k->ov_vec.v_count[i] = sizeof(*k); - p->k->ov_buf[i] = m0_alloc(sizeof(*k)); - memcpy(p->k->ov_buf[i], &k[i], sizeof(*k)); + p->k->ov_vec.v_count[i] = w->wit->key_size; + p->k->ov_buf[i] = m0_alloc(w->wit->key_size); + memcpy(p->k->ov_buf[i], (void*)kpart_one, kpart_one_size); + memcpy(p->k->ov_buf[i] + kpart_one_size, &k[i], sizeof(*k)); - crlog(CLL_DEBUG, "Generated k=" FID_F, FID_P(&k[i])); + crlog(CLL_DEBUG, "Generated k=%s:" FID_F, kpart_one, FID_P(&k[i])); } - return M0_RC(0); } static int fill_kv_next(struct cr_idx_w *w, - struct m0_fid *k, struct kv_pair *p, size_t nr) + struct m0_fid *k, struct kv_pair *p, size_t nr, + int kpart_one_size, char *kpart_one) { p->k = idx_bufvec_alloc(nr); if (p->k == NULL) @@ -592,16 +721,18 @@ static int fill_kv_next(struct cr_idx_w *w, M0_ASSERT(p->k->ov_vec.v_nr == nr); M0_ASSERT(p->v->ov_vec.v_nr == nr); - p->k->ov_vec.v_count[0] = sizeof(*k); - p->k->ov_buf[0] = m0_alloc(sizeof(*k)); - memcpy(p->k->ov_buf[0], &k[0], sizeof(*k)); - crlog(CLL_DEBUG, "Generated next k=" FID_F, FID_P(&k[0])); + p->k->ov_vec.v_count[0] = w->wit->key_size; + p->k->ov_buf[0] = m0_alloc(w->wit->key_size); + memcpy(p->k->ov_buf[0], (void*)kpart_one, kpart_one_size); + memcpy(p->k->ov_buf[0] + kpart_one_size, &k[0], sizeof(*k)); + crlog(CLL_DEBUG, "Generated next k=%s:" FID_F, kpart_one, FID_P(&k[0])); return M0_RC(0); } static int fill_kv_get(struct cr_idx_w *w, - struct m0_fid *k, struct kv_pair *p, size_t nr) + struct m0_fid *k, struct kv_pair *p, size_t nr, + int kpart_one_size, char *kpart_one) { int i; @@ -619,17 +750,18 @@ static int fill_kv_get(struct cr_idx_w *w, M0_ASSERT(p->v->ov_vec.v_nr == nr); for (i = 0; i < nr; i++) { - p->k->ov_vec.v_count[i] = sizeof(*k); - p->k->ov_buf[i] = m0_alloc(sizeof(*k)); - memcpy(p->k->ov_buf[i], &k[i], sizeof(*k)); - crlog(CLL_DEBUG, "Generated k=" FID_F, FID_P(&k[i])); + p->k->ov_vec.v_count[i] = w->wit->key_size; + p->k->ov_buf[i] = m0_alloc(w->wit->key_size); + memcpy(p->k->ov_buf[i], (void*)kpart_one, kpart_one_size); + memcpy(p->k->ov_buf[i] + kpart_one_size, &k[i], sizeof(*k)); + crlog(CLL_DEBUG, "Generated k=%s:" FID_F, kpart_one, FID_P(&k[i])); } return M0_RC(0); } - static int fill_kv_put(struct cr_idx_w *w, - struct m0_fid *k, struct kv_pair *p, size_t nr) + struct m0_fid *k, struct kv_pair *p, size_t nr, + int kpart_one_size, char *kpart_one) { int vlen; int i; @@ -648,14 +780,16 @@ static int fill_kv_put(struct cr_idx_w *w, M0_ASSERT(p->v->ov_vec.v_nr == nr); for (i = 0; i < nr; i++) { - p->k->ov_vec.v_count[i] = sizeof(*k); - p->k->ov_buf[i] = m0_alloc_aligned(sizeof(*k), PAGE_SHIFT); - memcpy(p->k->ov_buf[i], &k[i], sizeof(*k)); + p->k->ov_vec.v_count[i] = w->wit->key_size; + p->k->ov_buf[i] = m0_alloc_aligned(w->wit->key_size, PAGE_SHIFT); + memcpy(p->k->ov_buf[i], (void*)kpart_one, kpart_one_size); + memcpy(p->k->ov_buf[i] + kpart_one_size, &k[i], sizeof(*k)); vlen = cr_idx_w_get_value_size(w); p->v->ov_vec.v_count[i] = vlen; p->v->ov_buf[i] = m0_alloc_aligned(vlen, PAGE_SHIFT); cr_get_random_string(p->v->ov_buf[i], vlen); - crlog(CLL_DEBUG, "Generated k=" FID_F ",v=%s", + crlog(CLL_DEBUG, "Generated k=%s:" FID_F ",v=%s", + kpart_one, FID_P(&k[i]), vlen > 128 ? "<...>": (char *) p->v->ov_buf[i]); } @@ -674,7 +808,9 @@ struct cr_idx_w_ops { int (*fill_kv)(struct cr_idx_w *w, struct m0_fid *k, struct kv_pair *p, - size_t nr); + size_t nr, + int kpart_one_size, + char *kpart_one); /** is op readonly? */ bool readonly; /** op filling bit (for !readonly) */ @@ -923,6 +1059,11 @@ static int cr_idx_w_execute(struct cr_idx_w *w, struct m0_fid *keys = NULL; struct kv_pair kv = {0}; int i; + /* key will contain */ + int kpart_one_size = w->wit->key_size - w->wit->min_key_size; + char kpart_one[kpart_one_size]; + m0_time_t op_start_time; + m0_time_t op_time; M0_PRE(nr_keys > 0); @@ -958,13 +1099,21 @@ static int cr_idx_w_execute(struct cr_idx_w *w, keys[i].f_container = w->key_prefix.f_container; } - rc = op->fill_kv(w, keys, &kv, nr_keys); + /* populating key prefix */ + memset(kpart_one, 'A', kpart_one_size); + + rc = op->fill_kv(w, keys, &kv, nr_keys, kpart_one_size, kpart_one); if (rc != 0) { rc = M0_ERR(rc); goto do_exit_kv; } - + /* accumulate time required by each op on opcode basis. */ + op_start_time = m0_time_now(); rc = cr_execute_query(&w->wit->index_fid, &kv, opcode); + op_time = m0_time_sub(m0_time_now(), op_start_time); + w->ciw_results.ciwr_ops_result[opcode].cior_ops_total_time_m0 = + m0_time_add(w->ciw_results.ciwr_ops_result[opcode].cior_ops_total_time_m0, + op_time); if (rc != 0) { rc = M0_ERR(rc); goto do_exit_kv; @@ -1078,7 +1227,7 @@ static int cr_idx_w_get_nr_remained_op_types(struct cr_idx_w *w) static void cr_idx_w_print_ops_table(struct cr_idx_w *w) { M0_PRE(ARRAY_SIZE(w->nr_ops) == 4); - crlog(CLL_TRACE, "prcnt: [%d, %d, %d, %d]", + crlog(CLL_TRACE, "ops remaining: [%d, %d, %d, %d]", w->nr_ops[0].nr, w->nr_ops[1].nr, w->nr_ops[2].nr, @@ -1371,7 +1520,8 @@ static int index_operation(struct workload *wt, do_exit_wg: cr_watchdog_fini(); cr_time_measure_end(&t); - cr_time_measure_report(&t, "all"); + cr_time_capture_results(&t, &w); + cr_time_measure_report(&t, w); do_exit: return M0_RC(rc); } diff --git a/motr/m0crate/parser.c b/motr/m0crate/parser.c index 2440d52ead0..a345fda426a 100644 --- a/motr/m0crate/parser.c +++ b/motr/m0crate/parser.c @@ -60,8 +60,6 @@ enum config_key_val { NR_OBJS, NUM_IDX, NUM_KVP, - RECORD_SIZE, - MAX_RSIZE, PUT, GET, NEXT, @@ -73,6 +71,10 @@ enum config_key_val { WARMUP_DEL_RATIO, KEY_PREFIX, KEY_ORDER, + KEY_SIZE, + VALUE_SIZE, + MAX_KEY_SIZE, + MAX_VALUE_SIZE, INDEX_FID, LOG_LEVEL, THREAD_OPS, @@ -116,8 +118,8 @@ struct key_lookup_table lookuptable[] = { {"OPS", NUM_OPS}, {"NUM_IDX", NUM_IDX}, {"NUM_KVP", NUM_KVP}, - {"RECORD_SIZE", RECORD_SIZE}, - {"MAX_RSIZE", MAX_RSIZE}, + {"RECORD_SIZE", VALUE_SIZE}, + {"MAX_RSIZE", MAX_VALUE_SIZE}, {"GET", GET}, {"PUT", PUT}, {"NEXT", NEXT}, @@ -129,6 +131,10 @@ struct key_lookup_table lookuptable[] = { {"WARMUP_DEL_RATIO", WARMUP_DEL_RATIO}, {"KEY_PREFIX", KEY_PREFIX}, {"KEY_ORDER", KEY_ORDER}, + {"KEY_SIZE", KEY_SIZE}, + {"VALUE_SIZE", VALUE_SIZE}, + {"MAX_KEY_SIZE", MAX_KEY_SIZE}, + {"MAX_VALUE_SIZE", MAX_VALUE_SIZE}, {"INDEX_FID", INDEX_FID}, {"LOG_LEVEL", LOG_LEVEL}, {"NR_OBJS", NR_OBJS}, @@ -357,19 +363,6 @@ int copy_value(struct workload *load, int max_workload, int *index, ciw = workload_index(w); ciw->num_kvs = atoi(value); break; - case RECORD_SIZE: - w = &load[*index]; - ciw = workload_index(w); - if (!strcmp(value, "random")) - ciw->record_size = -1; - else - ciw->record_size = parse_int_with_units(value, RECORD_SIZE); - break; - case MAX_RSIZE: - w = &load[*index]; - ciw = workload_index(w); - ciw->max_record_size = parse_int_with_units(value, MAX_RSIZE); - break; case PUT: w = &load[*index]; ciw = workload_index(w); @@ -442,6 +435,41 @@ int copy_value(struct workload *load, int max_workload, int *index, else parser_emit_error("Unkown key ordering: '%s'", value); break; + case KEY_SIZE: + w = &load[*index]; + ciw = workload_index(w); + if (strcmp(value, "random") == 0) + ciw->key_size = -1; + else + ciw->key_size = parse_int(value, KEY_SIZE); + break; + case VALUE_SIZE: + w = &load[*index]; + ciw = workload_index(w); + if (strcmp(value, "random") == 0) + ciw->value_size = -1; + else { + ciw->value_size = parse_int(value, VALUE_SIZE); + if (strcmp(key, "RECORD_SIZE") == 0) { + cr_log(CLL_WARN, "RECORD_SIZE is being deprecated, use KEY_SIZE and VALUE_SIZE.\n"); + ciw->value_size = ciw->value_size - ciw->key_size; + } + } + break; + case MAX_KEY_SIZE: + w = &load[*index]; + ciw = workload_index(w); + ciw->max_key_size = parse_int(value, MAX_KEY_SIZE); + break; + case MAX_VALUE_SIZE: + w = &load[*index]; + ciw = workload_index(w); + ciw->max_value_size = parse_int(value, MAX_VALUE_SIZE); + if (strcmp(key, "MAX_RSIZE") == 0) { + cr_log(CLL_WARN, "MAX_RSIZE is being deprecated, use MAX_KEY_SIZE and MAX_VALUE_SIZE.\n"); + ciw->max_value_size = ciw->max_value_size - ciw->max_key_size; + } + break; case INDEX_FID: w = &load[*index]; ciw = workload_index(w); diff --git a/motr/m0crate/scripts/README.md b/motr/m0crate/scripts/README.md new file mode 100644 index 00000000000..24f417fc447 --- /dev/null +++ b/motr/m0crate/scripts/README.md @@ -0,0 +1,61 @@ +## Steps to take test runs on kvs (index) workload +We can use __m0crate__ client utility directly or __gen_index_yaml_run_workload__ script to take test runs on kvs workload. + +### 1. Run kvs workload using m0crate client utility + +You can use m0crate client utility to run single kvs workload. You can specify .yaml file as input to m0crate. +There are kvs .yaml template files available in __motr/m0crate/tests__ .yaml file contains 2 parts MOTR_CONFIG and WORKLOAD_SPEC. + +* MOTR_CONFIG contains motr cluster details, You would need to update these fields according to your cluster setup (see hctl status). +* WORKLOAD_SPEC contains workload details, You would need to update these fields according to your kvs workload requirement. + +```shell +[cortx-motr]$ ls motr/m0crate/tests/ +test1_io.yaml test1.yaml test2.yaml test3.yaml test4.yaml test5.yaml test6.yaml +[root@configs]# m0crate -S m0crate-index.yaml +``` + +### 2. Run kvs workloads using script +motr/m0crate/scripts dir contains __gen_index_yaml_run_workload__ and __m0crate-index.yaml.template__. __gen_index_yaml_run_workload__ is a script and __m0crate-index.yaml.template__ is a template file can be used as kvs workload template. + +```shell +[cortx-motr]$ ls motr/m0crate/scripts/ +gen_index_yaml_run_workload m0crate-index.yaml.template README.md +``` + +__gen_index_yaml_run_workload__ script can be used to generate and run workload in one go. Script __gen_index_yaml_run_workload__ uses __m0crate-index.yaml.template__ to generate workload .yaml files on the basis of __key_sizes__ and __value_sizes__ provided in the script. +__m0crate-index.yaml.template__ is used by above script as template, so before starting executing it make sure you have updated all the fields in yaml template file based on your requirements. + +* MOTR_LOCAL_ADDR, MOTR_HA_ADDR, PROF, PROCESS_FID these are the fields which are related to you motr cluster setup (see cmd: hctl status) +* You probably may need to update kvs workload spec fields too, NUM_KVP, NXRECORDS, OP_COUNT, KEY_ORDER etc. + +__Script execution steps__ + +1. Edit __m0crate-index.yaml.template__ file, Update __MOTR_CONFIG__ and __WORKLOAD_SPEC__ fields in the template. + +```shell +vim m0crate-index.yaml.template +``` + +2. Edit __gen_index_yaml_run_workload__ script file, Update __key_size__, __value_size__ and __start_of_indexfid__ values in the script. + +```shell +vim gen_index_yaml_run_workload +``` + +3. Execute __gen_index_yaml_run_workload__ script + +```shell +# ~/cortx-motr/motr/m0crate/scripts/gen_index_yaml_run_workload +``` + +4. After successful execution check execution logs and generated .yaml conf file in __workload_logs__ dir. Directory __workload_logs__ will have timestamp based directories. + +```shell +[temp]# ls workload_logs/ +010321-204231 010321-204704 +[temp]# ls workload_logs/010321-204704 +configs output +[temp]# ls workload_logs/010321-204704/output +kv_run_output.csv test_run.log +``` diff --git a/motr/m0crate/scripts/gen_index_yaml_run_workload b/motr/m0crate/scripts/gen_index_yaml_run_workload new file mode 100755 index 00000000000..cd1dc0a7e5e --- /dev/null +++ b/motr/m0crate/scripts/gen_index_yaml_run_workload @@ -0,0 +1,110 @@ +#!/usr/bin/bash +# +# Copyright (c) 2021 Seagate Technology LLC and/or its Affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# For any questions about this software or licensing, +# please email opensource@seagate.com or cortx-questions@seagate.com. +# + +# +# script: m0crate/scripts/gen_index_yaml_run_workload +# +# - script uses m0crate/scripts/m0crate-index.yaml.template as template +# and runs m0crate kv workload. +# - Update scripts/m0crate-index.yaml.template according to your usage. +# - you can specify multiple key_sizes and value_sizes based upon your requirement +# for example: keys=(16, 32, 64) and +# values=(1024, 2192, 4096) +# (Note: sizes are in bytes) +# - also specify uniq start_of_indexfid for each kv test run, later this is used as counter. +# - if you have 3*3 key_size and value_size combination in this situation +# there will be 9 workloads to run, +# - script will generate uniq indexfids by increamenting start_of_indexfid. In above 3*3 scenario +# consecutive 9 indexfids will get generated. +# +# - input: set key_sizes, value_sizes and start_of_indexfid +# - output: create workload_logs in curr dir, then creates timestamp base dir in which +# configs and output dir gets created. +# configs will have all .yaml files created in the workload execution by +# gen_index_yaml_run_workload script. +# outout will have workload execution logs in .csv and text format. + +script_dir=`dirname $(readlink -f $0)` +M0CRATE=$script_dir/../m0crate + +if [ ! -f "$M0CRATE" ]; then + echo "$M0CRATE not exists." + exit +fi + +keys=(64 128 256 512 1024) +values=(1024 4096 8192 16384 32768) +start_of_indexfid=0 + +time_dir=workload_logs/`date +%d%m%y-%H%M%S` +config_dir=$time_dir/configs +output_dir=$time_dir/output +mkdir -p $config_dir $output_dir + +log_csv_file=$output_dir/kv_run_output.csv +test_logs=$output_dir/test_run.log + +#method to execute kvs workload +function run_workload() { + klen=$1 + vlen=$2 + yaml_file=$3 + $M0CRATE -S $yaml_file 2>&1 | tee temp.log + cat temp.log >> $test_logs + test_time=`awk '/result: total_s/{ print $3 $5 $11","} + /result: PUT/{print $4 $6 $8","} + /result: GET/{print $4 $6 $8","} + /result: NEXT/{print $4 $6 $8","} + /result: DEL/{print $4 $6 $8""}' temp.log | tr '\n' ' '` + + echo "$klen, $vlen, $test_time" >> $log_csv_file + rm -f temp.log +} + +function generate_single_yaml_from_template() { + indexfid=$1 + klen=$2 + vlen=$3 + template_file="$script_dir/m0crate-index.yaml.template" + yaml_file="$config_dir/m0crate-index-$klen-$vlen.yaml" + sed "s/INDEXFID/$indexfid/g;s/KEYSIZE/$klen/g;s/VALUESIZE/$vlen/g" $template_file > $yaml_file + #echo "run_workload $klen $vlen $yaml_file" + run_workload $klen $vlen $yaml_file +} + +function generate_yaml_files_and_run_worklod() { + for klen in ${keys[@]} + do + for vlen in ${values[@]} + do + generate_single_yaml_from_template $start_of_indexfid $klen $vlen + start_of_indexfid="$((start_of_indexfid+1))" + done + done +} +function write_csv_header() { + echo -e "KEY_SIZE, VALUE_SIZE, TOTAL_TIME_S, TIME_PER_OP_NS, TOTAL_OPS," \ + "TOTAL_PUT_TIME_S, TIME_PER_PUT_OP_NS, PUT_OPS_CNT, " \ + "TOTAL_GET_TIME_S, TIME_PER_GET_OP_NS, GET_OPS_CNT, " \ + "TOTAL_NEXT_TIME_S, TIME_PER_NEXT_OP_NS, NEXT_OPS_CNT, " \ + "TOTAL_DEL_TIME_S, TIME_PER_DEL_OP_NS, DEL_OPS_CNT" >> $log_csv_file +} +write_csv_header +generate_yaml_files_and_run_worklod diff --git a/motr/m0crate/scripts/m0crate-index.yaml.template b/motr/m0crate/scripts/m0crate-index.yaml.template new file mode 100644 index 00000000000..fbaa2f8d114 --- /dev/null +++ b/motr/m0crate/scripts/m0crate-index.yaml.template @@ -0,0 +1,59 @@ +# +# Copyright (c) 2021 Seagate Technology LLC and/or its Affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# For any questions about this software or licensing, +# please email opensource@seagate.com or cortx-questions@seagate.com. +# + +# m0crate index .yaml +# Number of clients and server nodes - TBD. Layout with replication factor 1. +# Keys order: ordered. + +CrateConfig_Sections: [MOTR_CONFIG, WORKLOAD_SPEC] + +MOTR_CONFIG: + MOTR_LOCAL_ADDR: 192.168.52.53@tcp:12345:4:1 + MOTR_HA_ADDR: 192.168.52.53@tcp:12345:1:1 + PROF: <0x7000000000000001:0x37> # Profile fid + LAYOUT_ID: 1 # Defines the UNIT_SIZE (9: 1MB) + IS_OOSTORE: 1 # Is oostore-mode? + IS_READ_VERIFY: 0 # Enable read-verify? + TM_RECV_QUEUE_MIN_LEN: 16 # Minimum length of the receive queue + M0_MAX_RPC_MSG_SIZE: 262144 # Maximum rpc message size + PROCESS_FID: <0x7200000000000001:0x19> + IDX_SERVICE_ID: 1 + +WORKLOAD_SPEC: # Workload specification section + WORKLOAD: # First Workload + WORKLOAD_TYPE: 0 # Index(0), IO(1) + WORKLOAD_SEED: tstamp # SEED to the random number generator + NUM_KVP: 2 # int + NXRECORDS: 2 # int or default + OP_COUNT: 32 # int [units] or unlimited, total operations count. + EXEC_TIME: unlimited # int (seconds) or unlimited + WARMUP_PUT_CNT: 0 # int (ops) or all + WARMUP_DEL_RATIO: 0 # int (ops / ratio) + KEY_PREFIX: 0 # int or random + KEY_ORDER: ordered # ordered or random + KEY_SIZE: KEYSIZE # int or random + VALUE_SIZE: VALUESIZE # int or random + MAX_KEY_SIZE: 16384 # int + MAX_VALUE_SIZE: 32768 # int + INDEX_FID: <7800000000000001:INDEXFID> # fid + PUT: 60 # int (in percentage) + DEL: 10 # int (in percentage) + GET: 20 # int (in percentage) + NEXT: 10 # int (in percentage) + LOG_LEVEL: 2 # err(0), warn(1), info(2), trace(3), debug(4) diff --git a/motr/m0crate/tests/test1.yaml b/motr/m0crate/tests/test1.yaml index 4e560322717..9a7774af777 100644 --- a/motr/m0crate/tests/test1.yaml +++ b/motr/m0crate/tests/test1.yaml @@ -19,44 +19,45 @@ # Test case #1 - write speed (ordered keys) # Number of clients and server nodes - TBD. Layout with replication factor 1. -# Record value size is fixed - 4 Mb. +# Key size is fixed - 16 bytes +# Value size is fixed - 16 bytes # Keys order: ordered. -# Number of records 1 Tb/4 Mb/number of clients. # Only PUT requests. CrateConfig_Sections: [MOTR_CONFIG, WORKLOAD_SPEC] MOTR_CONFIG: - MOTR_LOCAL_ADDR: 10.0.2.15@tcp:12345:34:123 -  MOTR_HA_ADDR: 10.0.2.15@tcp:12345:34:101 -  PROF: <0x7000000000000001:1> -  LAYOUT_ID: 1 -  BLOCK_SIZE: 1048576 -  IS_OOSTORE: 1 -  IS_READ_VERIFY: 0 -  TM_RECV_QUEUE_MIN_LEN: 2 -  MAX_RPC_MSG_SIZE: 131072 -  PROCESS_FID: <0x7200000000000000:0> -  IDX_SERVICE_ID: 1 -  CASS_CLUSTER_EP: "127.0.0.1" -  CASS_KEYSPACE: "motr_index_keyspace" -  CASS_MAX_COL_FAMILY_NUM: 1 + MOTR_LOCAL_ADDR: 192.168.52.53@tcp:12345:4:1 + MOTR_HA_ADDR: 192.168.52.53@tcp:12345:1:1 + PROF: <0x7000000000000001:0x37> + LAYOUT_ID: 1 + IS_OOSTORE: 1 + IS_READ_VERIFY: 0 + TM_RECV_QUEUE_MIN_LEN: 2 + M0_MAX_RPC_MSG_SIZE: 131072 + PROCESS_FID: <0x7200000000000001:0x19> + IDX_SERVICE_ID: 1 + CASS_CLUSTER_EP: "127.0.0.1" + CASS_KEYSPACE: "motr_index_keyspace" + CASS_MAX_COL_FAMILY_NUM: 1 WORKLOAD_SPEC: - WORKLOAD_TYPE: 0 - WORKLOAD_SEED: tstamp - NUM_KVP: 8 - NXRECORDS: default # int or default - RECORD_SIZE: 32 # int [units] or random - MAX_RSIZE: 1M # int [units] - OP_COUNT: 32 # int [units] or unlimited = (2 ** 31 - 1) / (128 * NUM_KVP) - EXEC_TIME: unlimited # int (seconds) or unlimited - WARMUP_PUT_CNT: 0 # int (ops) or all - WARMUP_DEL_RATIO: 0 # int (ops / ratio) - KEY_PREFIX: random # int - KEY_ORDER: ordered # ordered or random - INDEX_FID: <7800000000000001:0> # fid - PUT: 100 # int - DEL: 0 # int - GET: 0 # int - NEXT: 0 # int - LOG_LEVEL: 4 # err(0), warn(1), info(2), trace(3), debug(4) + WORKLOAD_TYPE: 0 + WORKLOAD_SEED: tstamp + NUM_KVP: 8 + NXRECORDS: default # int or default + KEY_SIZE: 16 # int [units] or random + VALUE_SIZE: 16 # int [units] or random + MAX_KEY_SIZE: 512K # int [units] + MAX_VALUE_SIZE: 512K # int [units] + OP_COUNT: 32 # int [units] or unlimited = (2 ** 31 - 1) / (128 * NUM_KVP) + EXEC_TIME: unlimited # int (seconds) or unlimited + WARMUP_PUT_CNT: 0 # int (ops) or all + WARMUP_DEL_RATIO: 0 # int (ops / ratio) + KEY_PREFIX: random # int + KEY_ORDER: ordered # ordered or random + INDEX_FID: <7800000000000001:0> # fid + PUT: 100 # int + DEL: 0 # int + GET: 0 # int + NEXT: 0 # int + LOG_LEVEL: 4 # err(0), warn(1), info(2), trace(3), debug(4) diff --git a/motr/m0crate/tests/test2.yaml b/motr/m0crate/tests/test2.yaml index 00a58d617c2..6de7f5d2dd9 100644 --- a/motr/m0crate/tests/test2.yaml +++ b/motr/m0crate/tests/test2.yaml @@ -19,44 +19,45 @@ # Test case #2 - read speed (ordered keys) # Number of clients and server nodes - TBD. Layout with replication factor 1. -# Record value size is fixed - 4 Mb. +# Key size is fixed - 16 bytes +# Value size is fixed - 16 bytes # Keys order: ordered. -# Number of records 1 Tb/4 Mb/number of clients. # Initial number of records 1 Tb/4 Mb/number of clients (populate the cluster with data). CrateConfig_Sections: [MOTR_CONFIG, WORKLOAD_SPEC] MOTR_CONFIG: - MOTR_LOCAL_ADDR: 10.0.2.15@tcp:12345:34:123 -  MOTR_HA_ADDR: 10.0.2.15@tcp:12345:34:101 -  PROF: <0x7000000000000001:1> -  LAYOUT_ID: 1 -  BLOCK_SIZE: 1048576 -  IS_OOSTORE: 1 -  IS_READ_VERIFY: 0 -  TM_RECV_QUEUE_MIN_LEN: 2 -  MAX_RPC_MSG_SIZE: 131072 -  PROCESS_FID: <0x7200000000000000:0> -  IDX_SERVICE_ID: 1 -  CASS_CLUSTER_EP: "127.0.0.1" -  CASS_KEYSPACE: "motr_index_keyspace" -  CASS_MAX_COL_FAMILY_NUM: 1 + MOTR_LOCAL_ADDR: 192.168.52.53@tcp:12345:4:1 + MOTR_HA_ADDR: 192.168.52.53@tcp:12345:1:1 + PROF: <0x7000000000000001:0x37> + LAYOUT_ID: 1 + IS_OOSTORE: 1 + IS_READ_VERIFY: 0 + TM_RECV_QUEUE_MIN_LEN: 2 + M0_MAX_RPC_MSG_SIZE: 131072 + PROCESS_FID: <0x7200000000000001:0x19> + IDX_SERVICE_ID: 1 + CASS_CLUSTER_EP: "127.0.0.1" + CASS_KEYSPACE: "motr_index_keyspace" + CASS_MAX_COL_FAMILY_NUM: 1 WORKLOAD_SPEC: - WORKLOAD_TYPE: 0 - WORKLOAD_SEED: tstamp - NUM_KVP: 8 - NXRECORDS: default # int or default - RECORD_SIZE: 32 # int [units] or random - MAX_RSIZE: 1M # int [units] - OP_COUNT: 32 # int [units] or unlimited = (2 ** 31 - 1) / (128 * NUM_KVP) - EXEC_TIME: unlimited # int (seconds) or unlimited - WARMUP_PUT_CNT: all # int (ops) or all - WARMUP_DEL_RATIO: 0 # int (ops / ratio) - KEY_PREFIX: random # int - KEY_ORDER: ordered # ordered or random - INDEX_FID: <7800000000000001:0> # fid - PUT: 0 # int - DEL: 0 # int - GET: 100 # int - NEXT: 0 # int - LOG_LEVEL: 4 # err(0), warn(1), info(2), trace(3), debug(4) + WORKLOAD_TYPE: 0 + WORKLOAD_SEED: tstamp + NUM_KVP: 8 + NXRECORDS: default # int or default + KEY_SIZE: 16 # int [units] or random + VALUE_SIZE: 16 # int [units] or random + MAX_KEY_SIZE: 512K # int [units] + MAX_VALUE_SIZE: 512K # int [units] + OP_COUNT: 32 # int [units] or unlimited = (2 ** 31 - 1) / (128 * NUM_KVP) + EXEC_TIME: unlimited # int (seconds) or unlimited + WARMUP_PUT_CNT: all # int (ops) or all + WARMUP_DEL_RATIO: 0 # int (ops / ratio) + KEY_PREFIX: random # int + KEY_ORDER: ordered # ordered or random + INDEX_FID: <7800000000000001:0> # fid + PUT: 0 # int + DEL: 0 # int + GET: 100 # int + NEXT: 0 # int + LOG_LEVEL: 4 # err(0), warn(1), info(2), trace(3), debug(4) diff --git a/motr/m0crate/tests/test3.yaml b/motr/m0crate/tests/test3.yaml index 60f01259b26..60abfd2c866 100644 --- a/motr/m0crate/tests/test3.yaml +++ b/motr/m0crate/tests/test3.yaml @@ -19,44 +19,45 @@ # Test case #3 - write speed (unordered keys) # Number of clients and server nodes - TBD. Layout with replication factor 1. -# Record value size is fixed - 4 Mb. +# Key size is fixed - 16 bytes +# Value size is fixed - 16 bytes # Keys order: random. -# Number of records 1 Tb/4 Mb/number of clients. # Only PUT requests. CrateConfig_Sections: [MOTR_CONFIG, WORKLOAD_SPEC] MOTR_CONFIG: - MOTR_LOCAL_ADDR: 10.0.2.15@tcp:12345:34:123 -  MOTR_HA_ADDR: 10.0.2.15@tcp:12345:34:101 -  PROF: <0x7000000000000001:1> -  LAYOUT_ID: 1 -  BLOCK_SIZE: 1048576 -  IS_OOSTORE: 1 -  IS_READ_VERIFY: 0 -  TM_RECV_QUEUE_MIN_LEN: 2 -  MAX_RPC_MSG_SIZE: 131072 -  PROCESS_FID: <0x7200000000000000:0> -  IDX_SERVICE_ID: 1 -  CASS_CLUSTER_EP: "127.0.0.1" -  CASS_KEYSPACE: "motr_index_keyspace" -  CASS_MAX_COL_FAMILY_NUM: 1 + MOTR_LOCAL_ADDR: 192.168.52.53@tcp:12345:4:1 + MOTR_HA_ADDR: 192.168.52.53@tcp:12345:1:1 + PROF: <0x7000000000000001:0x37> + LAYOUT_ID: 1 + IS_OOSTORE: 1 + IS_READ_VERIFY: 0 + TM_RECV_QUEUE_MIN_LEN: 2 + M0_MAX_RPC_MSG_SIZE: 131072 + PROCESS_FID: <0x7200000000000001:0x19> + IDX_SERVICE_ID: 1 + CASS_CLUSTER_EP: "127.0.0.1" + CASS_KEYSPACE: "motr_index_keyspace" + CASS_MAX_COL_FAMILY_NUM: 1 WORKLOAD_SPEC: - WORKLOAD_TYPE: 0 - WORKLOAD_SEED: tstamp - NUM_KVP: 8 - NXRECORDS: default # int or default - RECORD_SIZE: 32 # int [units] or random - MAX_RSIZE: 1M # int [units] - OP_COUNT: 32 # int [units] or unlimited = (2 ** 31 - 1) / (128 * NUM_KVP) - EXEC_TIME: unlimited # int (seconds) or unlimited - WARMUP_PUT_CNT: 0 # int (ops) or all - WARMUP_DEL_RATIO: 0 # int (ops / ratio) - KEY_PREFIX: random # int - KEY_ORDER: random # ordered or random - INDEX_FID: <7800000000000001:0> # fid - PUT: 100 # int - DEL: 0 # int - GET: 0 # int - NEXT: 0 # int - LOG_LEVEL: 4 # err(0), warn(1), info(2), trace(3), debug(4) + WORKLOAD_TYPE: 0 + WORKLOAD_SEED: tstamp + NUM_KVP: 8 + NXRECORDS: default # int or default + KEY_SIZE: 16 # int [units] or random + VALUE_SIZE: 16 # int [units] or random + MAX_KEY_SIZE: 512K # int [units] + MAX_VALUE_SIZE: 512K # int [units] + OP_COUNT: 32 # int [units] or unlimited = (2 ** 31 - 1) / (128 * NUM_KVP) + EXEC_TIME: unlimited # int (seconds) or unlimited + WARMUP_PUT_CNT: 0 # int (ops) or all + WARMUP_DEL_RATIO: 0 # int (ops / ratio) + KEY_PREFIX: random # int + KEY_ORDER: random # ordered or random + INDEX_FID: <7800000000000001:0> # fid + PUT: 100 # int + DEL: 0 # int + GET: 0 # int + NEXT: 0 # int + LOG_LEVEL: 4 # err(0), warn(1), info(2), trace(3), debug(4) diff --git a/motr/m0crate/tests/test4.yaml b/motr/m0crate/tests/test4.yaml index f58ca637a40..b580cbdbc33 100644 --- a/motr/m0crate/tests/test4.yaml +++ b/motr/m0crate/tests/test4.yaml @@ -19,44 +19,45 @@ # Test case #4 - read speed (unordered keys) # Number of clients and server nodes - TBD. Layout with replication factor 1. -# Record value size is fixed - 4 Mb. +# Key size is fixed - 16 bytes +# Value size is fixed - 16 bytes # Keys order: random. -# Number of records 1 Tb/4 Mb/number of clients. # Initial number of records 1 Tb/4 Mb/number of clients (populate the cluster with data). CrateConfig_Sections: [MOTR_CONFIG, WORKLOAD_SPEC] MOTR_CONFIG: - MOTR_LOCAL_ADDR: 10.0.2.15@tcp:12345:34:123 -  MOTR_HA_ADDR: 10.0.2.15@tcp:12345:34:101 -  PROF: <0x7000000000000001:1> -  LAYOUT_ID: 1 -  BLOCK_SIZE: 1048576 -  IS_OOSTORE: 1 -  IS_READ_VERIFY: 0 -  TM_RECV_QUEUE_MIN_LEN: 2 -  MAX_RPC_MSG_SIZE: 131072 -  PROCESS_FID: <0x7200000000000000:0> -  IDX_SERVICE_ID: 1 -  CASS_CLUSTER_EP: "127.0.0.1" -  CASS_KEYSPACE: "motr_index_keyspace" -  CASS_MAX_COL_FAMILY_NUM: 1 + MOTR_LOCAL_ADDR: 192.168.52.53@tcp:12345:4:1 + MOTR_HA_ADDR: 192.168.52.53@tcp:12345:1:1 + PROF: <0x7000000000000001:0x37> + LAYOUT_ID: 1 + IS_OOSTORE: 1 + IS_READ_VERIFY: 0 + TM_RECV_QUEUE_MIN_LEN: 2 + M0_MAX_RPC_MSG_SIZE: 131072 + PROCESS_FID: <0x7200000000000001:0x19> + IDX_SERVICE_ID: 1 + CASS_CLUSTER_EP: "127.0.0.1" + CASS_KEYSPACE: "motr_index_keyspace" + CASS_MAX_COL_FAMILY_NUM: 1 WORKLOAD_SPEC: - WORKLOAD_TYPE: 0 - WORKLOAD_SEED: tstamp - NUM_KVP: 8 - NXRECORDS: default # int or default - RECORD_SIZE: 32 # int [units] or random - MAX_RSIZE: 1M # int [units] - OP_COUNT: 32 # int [units] or unlimited = (2 ** 31 - 1) / (128 * NUM_KVP) - EXEC_TIME: unlimited # int (seconds) or unlimited - WARMUP_PUT_CNT: all # int (ops) or all - WARMUP_DEL_RATIO: 0 # int (ops / ratio) - KEY_PREFIX: random # int - KEY_ORDER: random # ordered or random - INDEX_FID: <7800000000000001:0> # fid - PUT: 0 # int - DEL: 0 # int - GET: 100 # int - NEXT: 0 # int - LOG_LEVEL: 4 # err(0), warn(1), info(2), trace(3), debug(4) + WORKLOAD_TYPE: 0 + WORKLOAD_SEED: tstamp + NUM_KVP: 8 + NXRECORDS: default # int or default + KEY_SIZE: 16 # int [units] or random + VALUE_SIZE: 16 # int [units] or random + MAX_KEY_SIZE: 512K # int [units] + MAX_VALUE_SIZE: 512K # int [units] + OP_COUNT: 32 # int [units] or unlimited = (2 ** 31 - 1) / (128 * NUM_KVP) + EXEC_TIME: unlimited # int (seconds) or unlimited + WARMUP_PUT_CNT: all # int (ops) or all + WARMUP_DEL_RATIO: 0 # int (ops / ratio) + KEY_PREFIX: random # int + KEY_ORDER: random # ordered or random + INDEX_FID: <7800000000000001:0> # fid + PUT: 0 # int + DEL: 0 # int + GET: 100 # int + NEXT: 0 # int + LOG_LEVEL: 4 # err(0), warn(1), info(2), trace(3), debug(4) diff --git a/motr/m0crate/tests/test5.yaml b/motr/m0crate/tests/test5.yaml index f5a8036d664..d86ead5cd8b 100644 --- a/motr/m0crate/tests/test5.yaml +++ b/motr/m0crate/tests/test5.yaml @@ -19,44 +19,45 @@ # Test case #5 - write speed (unordered keys, fragmented segment) # Number of clients and server nodes - TBD. Layout with replication factor 1. -# Record value size is fixed - 4 Mb. +# Key size is fixed - 16 bytes +# Value size is fixed - 16 bytes # Keys order: random. -# Number of records 1 Tb/4 Mb/number of clients. # Only PUT requests. CrateConfig_Sections: [MOTR_CONFIG, WORKLOAD_SPEC] MOTR_CONFIG: - MOTR_LOCAL_ADDR: 10.0.2.15@tcp:12345:34:123 -  MOTR_HA_ADDR: 10.0.2.15@tcp:12345:34:101 -  PROF: <0x7000000000000001:1> -  LAYOUT_ID: 1 -  BLOCK_SIZE: 1048576 -  IS_OOSTORE: 1 -  IS_READ_VERIFY: 0 -  TM_RECV_QUEUE_MIN_LEN: 2 -  MAX_RPC_MSG_SIZE: 131072 -  PROCESS_FID: <0x7200000000000000:0> -  IDX_SERVICE_ID: 1 -  CASS_CLUSTER_EP: "127.0.0.1" -  CASS_KEYSPACE: "motr_index_keyspace" -  CASS_MAX_COL_FAMILY_NUM: 1 + MOTR_LOCAL_ADDR: 192.168.52.53@tcp:12345:4:1 + MOTR_HA_ADDR: 192.168.52.53@tcp:12345:1:1 + PROF: <0x7000000000000001:0x37> + LAYOUT_ID: 1 + IS_OOSTORE: 1 + IS_READ_VERIFY: 0 + TM_RECV_QUEUE_MIN_LEN: 2 + M0_MAX_RPC_MSG_SIZE: 131072 + PROCESS_FID: <0x7200000000000001:0x19> + IDX_SERVICE_ID: 1 + CASS_CLUSTER_EP: "127.0.0.1" + CASS_KEYSPACE: "motr_index_keyspace" + CASS_MAX_COL_FAMILY_NUM: 1 WORKLOAD_SPEC: - WORKLOAD_TYPE: 0 - WORKLOAD_SEED: tstamp - NUM_KVP: 8 - NXRECORDS: default # int or default - RECORD_SIZE: 32 # int [units] or random - MAX_RSIZE: 1M # int [units] - OP_COUNT: 32 # int [units] or unlimited = (2 ** 31 - 1) / (128 * NUM_KVP) - EXEC_TIME: unlimited # int (seconds) or unlimited - WARMUP_PUT_CNT: 64 # int (ops) or all - WARMUP_DEL_RATIO: 2 # int (ops / ratio) - KEY_PREFIX: random # int - KEY_ORDER: random # ordered or random - INDEX_FID: <7800000000000001:0> # fid - PUT: 100 # int - DEL: 0 # int - GET: 0 # int - NEXT: 0 # int - LOG_LEVEL: 4 # err(0), warn(1), info(2), trace(3), debug(4) + WORKLOAD_TYPE: 0 + WORKLOAD_SEED: tstamp + NUM_KVP: 8 + NXRECORDS: default # int or default + KEY_SIZE: 16 # int [units] or random + VALUE_SIZE: 16 # int [units] or random + MAX_KEY_SIZE: 512K # int [units] + MAX_VALUE_SIZE: 512K # int [units] + OP_COUNT: 32 # int [units] or unlimited = (2 ** 31 - 1) / (128 * NUM_KVP) + EXEC_TIME: unlimited # int (seconds) or unlimited + WARMUP_PUT_CNT: 64 # int (ops) or all + WARMUP_DEL_RATIO: 2 # int (ops / ratio) + KEY_PREFIX: random # int + KEY_ORDER: random # ordered or random + INDEX_FID: <7800000000000001:0> # fid + PUT: 100 # int + DEL: 0 # int + GET: 0 # int + NEXT: 0 # int + LOG_LEVEL: 4 # err(0), warn(1), info(2), trace(3), debug(4) diff --git a/motr/m0crate/tests/test6.yaml b/motr/m0crate/tests/test6.yaml index 35ece70cea2..f7db3545e67 100644 --- a/motr/m0crate/tests/test6.yaml +++ b/motr/m0crate/tests/test6.yaml @@ -19,44 +19,45 @@ # Test case #6 - read speed (unordered keys, fragmented segment) # Number of clients and server nodes - TBD. Layout with replication factor 1. -# Record value size is fixed - 4 Mb. +# Key size is fixed - 16 bytes +# Value size is fixed - 16 bytes # Keys order: random. -# Number of records 1 Tb/4 Mb/number of clients. # Initial number of records 1 Tb/4 Mb/number of clients (populate the cluster with data). CrateConfig_Sections: [MOTR_CONFIG, WORKLOAD_SPEC] MOTR_CONFIG: - MOTR_LOCAL_ADDR: 10.0.2.15@tcp:12345:34:123 -  MOTR_HA_ADDR: 10.0.2.15@tcp:12345:34:101 -  PROF: <0x7000000000000001:1> -  LAYOUT_ID: 1 -  BLOCK_SIZE: 1048576 -  IS_OOSTORE: 1 -  IS_READ_VERIFY: 0 -  TM_RECV_QUEUE_MIN_LEN: 2 -  MAX_RPC_MSG_SIZE: 131072 -  PROCESS_FID: <0x7200000000000000:0> -  IDX_SERVICE_ID: 1 -  CASS_CLUSTER_EP: "127.0.0.1" -  CASS_KEYSPACE: "motr_index_keyspace" -  CASS_MAX_COL_FAMILY_NUM: 1 + MOTR_LOCAL_ADDR: 192.168.52.53@tcp:12345:4:1 + MOTR_HA_ADDR: 192.168.52.53@tcp:12345:1:1 + PROF: <0x7000000000000001:0x37> + LAYOUT_ID: 1 + IS_OOSTORE: 1 + IS_READ_VERIFY: 0 + TM_RECV_QUEUE_MIN_LEN: 2 + M0_MAX_RPC_MSG_SIZE: 131072 + PROCESS_FID: <0x7200000000000001:0x19> + IDX_SERVICE_ID: 1 + CASS_CLUSTER_EP: "127.0.0.1" + CASS_KEYSPACE: "motr_index_keyspace" + CASS_MAX_COL_FAMILY_NUM: 1 WORKLOAD_SPEC: - WORKLOAD_TYPE: 0 - WORKLOAD_SEED: tstamp - NUM_KVP: 8 - NXRECORDS: default # int or default - RECORD_SIZE: 32 # int [units] or random - MAX_RSIZE: 1M # int [units] - OP_COUNT: 32 # int [units] or unlimited = (2 ** 31 - 1) / (128 * NUM_KVP) - EXEC_TIME: unlimited # int (seconds) or unlimited - WARMUP_PUT_CNT: all # int (ops) or all - WARMUP_DEL_RATIO: 2 # int (ops / ratio) - KEY_PREFIX: random # int - KEY_ORDER: random # ordered or random - INDEX_FID: <7800000000000001:0> # fid - PUT: 0 # int - DEL: 0 # int - GET: 100 # int - NEXT: 0 # int - LOG_LEVEL: 4 # err(0), warn(1), info(2), trace(3), debug(4) + WORKLOAD_TYPE: 0 + WORKLOAD_SEED: tstamp + NUM_KVP: 8 + NXRECORDS: default # int or default + KEY_SIZE: 16 # int [units] or random + VALUE_SIZE: 16 # int [units] or random + MAX_KEY_SIZE: 512K # int [units] + MAX_VALUE_SIZE: 512K # int [units] + OP_COUNT: 32 # int [units] or unlimited = (2 ** 31 - 1) / (128 * NUM_KVP) + EXEC_TIME: unlimited # int (seconds) or unlimited + WARMUP_PUT_CNT: all # int (ops) or all + WARMUP_DEL_RATIO: 2 # int (ops / ratio) + KEY_PREFIX: random # int + KEY_ORDER: random # ordered or random + INDEX_FID: <7800000000000001:0> # fid + PUT: 0 # int + DEL: 0 # int + GET: 100 # int + NEXT: 0 # int + LOG_LEVEL: 4 # err(0), warn(1), info(2), trace(3), debug(4)