From 54cca7da76f14045720b262dcd06f840cdbf5032 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Thu, 14 Jan 2021 17:41:56 +0000 Subject: [PATCH 1/2] [add] Enable mixing key placeholder with string arguments --- client.cpp | 11 ++++-- protocol.cpp | 5 --- tests/test_requirements.txt | 2 -- tests/tests_oss_simple_flow.py | 61 ++++++++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 9 deletions(-) diff --git a/client.cpp b/client.cpp index 82045ed0..6b54afaf 100755 --- a/client.cpp +++ b/client.cpp @@ -50,6 +50,7 @@ #include "client.h" #include "cluster_client.h" +#include "config_types.h" bool client::setup_client(benchmark_config *config, abstract_protocol *protocol, object_generator *objgen) @@ -256,8 +257,14 @@ void client::create_arbitrary_request(const arbitrary_command* cmd, struct timev assert(key != NULL); assert(key_len > 0); - - cmd_size += m_connections[conn_id]->send_arbitrary_command(arg, key, key_len); + //when we have static data mixed with the key placeholder + if (arg->data.length() != strlen(KEY_PLACEHOLDER)) { + std::string str (arg->data); + str.replace(str.find(KEY_PLACEHOLDER),strlen(KEY_PLACEHOLDER),key); + cmd_size += m_connections[conn_id]->send_arbitrary_command(arg, str.c_str(), str.length() ); + } else{ + cmd_size += m_connections[conn_id]->send_arbitrary_command(arg, key, key_len); + } } else if (arg->type == data_type) { unsigned int value_len; const char *value = m_obj_gen->get_value(0, &value_len); diff --git a/protocol.cpp b/protocol.cpp index 51ac53a1..b922a302 100644 --- a/protocol.cpp +++ b/protocol.cpp @@ -631,11 +631,6 @@ bool redis_protocol::format_arbitrary_command(arbitrary_command &cmd) { // check arg type if (current_arg->data.find(KEY_PLACEHOLDER) != std::string::npos) { - if (current_arg->data.length() != strlen(KEY_PLACEHOLDER)) { - benchmark_error_log("error: key placeholder can't combined with other data\n"); - return false; - } - current_arg->type = key_type; } else if (current_arg->data.find(DATA_PLACEHOLDER) != std::string::npos) { if (current_arg->data.length() != strlen(DATA_PLACEHOLDER)) { diff --git a/tests/test_requirements.txt b/tests/test_requirements.txt index 8a689bff..41deaa97 100644 --- a/tests/test_requirements.txt +++ b/tests/test_requirements.txt @@ -1,4 +1,2 @@ -redis>=3.0.0 -git+https://github.com/Grokzen/redis-py-cluster.git@master git+https://github.com/RedisLabsModules/RLTest.git@master git+https://github.com/RedisLabs/mbdirector.git@master \ No newline at end of file diff --git a/tests/tests_oss_simple_flow.py b/tests/tests_oss_simple_flow.py index 0d29fb03..5447990e 100644 --- a/tests/tests_oss_simple_flow.py +++ b/tests/tests_oss_simple_flow.py @@ -144,3 +144,64 @@ def test_default_set_get_3_runs(env): overall_request_count = agg_info_commandstats(master_nodes_connections, merged_command_stats) assert_minimum_memtier_outcomes(config, env, memtier_ok, merged_command_stats, overall_expected_request_count, overall_request_count) + + +def test_key_placeholder(env): + env.skipOnCluster() + run_count = 1 + benchmark_specs = {"name": env.testName, "args": ['--command=HSET __key__ f __data__']} + addTLSArgs(benchmark_specs, env) + config = get_default_memtier_config() + master_nodes_list = env.getMasterNodesList() + overall_expected_request_count = get_expected_request_count(config) * run_count + + add_required_env_arguments(benchmark_specs, config, env, master_nodes_list) + + # Create a temporary directory + test_dir = tempfile.mkdtemp() + + config = RunConfig(test_dir, env.testName, config, {}) + ensure_clean_benchmark_folder(config.results_dir) + + benchmark = Benchmark.from_json(config, benchmark_specs) + + # benchmark.run() returns True if the return code of memtier_benchmark was 0 + memtier_ok = benchmark.run() + debugPrintMemtierOnError(config, env, memtier_ok) + + master_nodes_connections = env.getOSSMasterNodesConnectionList() + merged_command_stats = {'cmdstat_hset': {'calls': 0}} + overall_request_count = agg_info_commandstats(master_nodes_connections, merged_command_stats) + assert_minimum_memtier_outcomes(config, env, memtier_ok, merged_command_stats, overall_expected_request_count, + overall_request_count) + + +# key placeholder combined with other data +def test_key_placeholder_togetherwithdata(env): + env.skipOnCluster() + run_count = 1 + benchmark_specs = {"name": env.testName, "args": ['--command=HSET prefix:__key__:suffix f __data__']} + addTLSArgs(benchmark_specs, env) + config = get_default_memtier_config() + master_nodes_list = env.getMasterNodesList() + overall_expected_request_count = get_expected_request_count(config) * run_count + + add_required_env_arguments(benchmark_specs, config, env, master_nodes_list) + + # Create a temporary directory + test_dir = tempfile.mkdtemp() + + config = RunConfig(test_dir, env.testName, config, {}) + ensure_clean_benchmark_folder(config.results_dir) + + benchmark = Benchmark.from_json(config, benchmark_specs) + + # benchmark.run() returns True if the return code of memtier_benchmark was 0 + memtier_ok = benchmark.run() + debugPrintMemtierOnError(config, env, memtier_ok) + + master_nodes_connections = env.getOSSMasterNodesConnectionList() + merged_command_stats = {'cmdstat_hset': {'calls': 0}} + overall_request_count = agg_info_commandstats(master_nodes_connections, merged_command_stats) + assert_minimum_memtier_outcomes(config, env, memtier_ok, merged_command_stats, overall_expected_request_count, + overall_request_count) From 85759ab3bd77748d4a79f8e5ec5c9ac29bda3210 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Mon, 3 May 2021 13:41:43 +0100 Subject: [PATCH 2/2] [fix] Fixes per PR review: avoid str::replace during runtime and using precomputed prefix and suffix --- client.cpp | 6 ++++-- config_types.h | 3 +++ protocol.cpp | 14 +++++++++++++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/client.cpp b/client.cpp index 6b54afaf..4744f3b7 100755 --- a/client.cpp +++ b/client.cpp @@ -46,6 +46,7 @@ #include #include +#include #include #include "client.h" @@ -260,8 +261,9 @@ void client::create_arbitrary_request(const arbitrary_command* cmd, struct timev //when we have static data mixed with the key placeholder if (arg->data.length() != strlen(KEY_PLACEHOLDER)) { std::string str (arg->data); - str.replace(str.find(KEY_PLACEHOLDER),strlen(KEY_PLACEHOLDER),key); - cmd_size += m_connections[conn_id]->send_arbitrary_command(arg, str.c_str(), str.length() ); + std::ostringstream key_stream; + key_stream << arg->data_prefix << key << arg->data_suffix; + cmd_size += m_connections[conn_id]->send_arbitrary_command(arg, key_stream.str().c_str(), key_stream.str().length()); } else{ cmd_size += m_connections[conn_id]->send_arbitrary_command(arg, key, key_len); } diff --git a/config_types.h b/config_types.h index b2a92239..73b541c4 100644 --- a/config_types.h +++ b/config_types.h @@ -116,6 +116,9 @@ struct command_arg { command_arg(const char* arg, unsigned int arg_len) : type(undefined_type), data(arg, arg_len) {;} command_arg_type type; std::string data; + // the prefix and suffix strings are used for mixed key placeholder storing of substrings + std::string data_prefix; + std::string data_suffix; }; struct arbitrary_command { diff --git a/protocol.cpp b/protocol.cpp index b922a302..fa4bbf23 100644 --- a/protocol.cpp +++ b/protocol.cpp @@ -630,8 +630,20 @@ bool redis_protocol::format_arbitrary_command(arbitrary_command &cmd) { current_arg->type = const_type; // check arg type - if (current_arg->data.find(KEY_PLACEHOLDER) != std::string::npos) { + const std::size_t key_placeholder_start = current_arg->data.find(KEY_PLACEHOLDER); + if (key_placeholder_start != std::string::npos) { current_arg->type = key_type; + current_arg->data_prefix = ""; + current_arg->data_suffix = ""; + // check for prefix + if (key_placeholder_start > 0) { + current_arg->data_prefix = current_arg->data.substr(0,key_placeholder_start); + } + // check for sufix + const std::size_t suffix_start = strlen(KEY_PLACEHOLDER)+key_placeholder_start; + if (current_arg->data.length() > suffix_start) { + current_arg->data_suffix = current_arg->data.substr(suffix_start,current_arg->data.length()); + } } else if (current_arg->data.find(DATA_PLACEHOLDER) != std::string::npos) { if (current_arg->data.length() != strlen(DATA_PLACEHOLDER)) { benchmark_error_log("error: data placeholder can't combined with other data\n");