Enabled per module configuration parameters

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redisbench-admin"
-version = "0.6.8"
+version = "0.6.9"
 description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "README.md"

redisbench_admin/commands/__init__.py

@@ -0,0 +1,5 @@
+#  BSD 3-Clause License
+#
+#  Copyright (c) 2022., Redis Labs Modules
+#  All rights reserved.
+#

redisbench_admin/commands/commands.json.py

@@ -0,0 +1,5 @@
+#  BSD 3-Clause License
+#
+#  Copyright (c) 2022., Redis Labs Modules
+#  All rights reserved.
+#

redisbench_admin/environments/oss_cluster.py

@@ -4,13 +4,12 @@
 #  All rights reserved.
 #
 import logging
-import os
 import subprocess
 from time import sleep
 
 import redis
 
-from redisbench_admin.utils.utils import wait_for_conn
+from redisbench_admin.utils.utils import wait_for_conn, redis_server_config_module_part
 
 
 def spin_up_local_redis_cluster(
@@ -21,6 +20,7 @@ def spin_up_local_redis_cluster(
     local_module_file,
     configuration_parameters=None,
     dataset_load_timeout_secs=60,
+    modules_configuration_parameters_map={},
 ):
     redis_processes = []
     redis_conns = []
@@ -29,7 +29,13 @@ def spin_up_local_redis_cluster(
         shard_port = master_shard_id + start_port - 1
 
         command = generate_cluster_redis_server_args(
-            dbdir, local_module_file, ip, shard_port, configuration_parameters, "no"
+            dbdir,
+            local_module_file,
+            ip,
+            shard_port,
+            configuration_parameters,
+            "no",
+            modules_configuration_parameters_map,
         )
 
         logging.info(
@@ -132,6 +138,7 @@ def generate_cluster_redis_server_args(
     port,
     configuration_parameters=None,
     daemonize="yes",
+    modules_configuration_parameters_map={},
 ):
     # start redis-server
     command = [
@@ -171,19 +178,13 @@ def generate_cluster_redis_server_args(
             )
     if local_module_file is not None:
         if type(local_module_file) == str:
-            command.extend(
-                [
-                    "--loadmodule",
-                    os.path.abspath(local_module_file),
-                ]
+            redis_server_config_module_part(
+                command, local_module_file, modules_configuration_parameters_map
             )
         if type(local_module_file) == list:
             for mod in local_module_file:
-                command.extend(
-                    [
-                        "--loadmodule",
-                        os.path.abspath(mod),
-                    ]
+                redis_server_config_module_part(
+                    command, mod, modules_configuration_parameters_map
                 )
     return command
 

redisbench_admin/environments/oss_standalone.py

@@ -4,12 +4,11 @@
 #  All rights reserved.
 #
 import logging
-import os
 import subprocess
 
 import redis
 
-from redisbench_admin.utils.utils import wait_for_conn
+from redisbench_admin.utils.utils import wait_for_conn, redis_server_config_module_part
 
 
 def spin_up_local_redis(
@@ -20,9 +19,15 @@ def spin_up_local_redis(
     configuration_parameters=None,
     dbdir_folder=None,
     dataset_load_timeout_secs=120,
+    modules_configuration_parameters_map={},
 ):
     command = generate_standalone_redis_server_args(
-        binary, dbdir, local_module_files, port, configuration_parameters
+        binary,
+        dbdir,
+        local_module_files,
+        port,
+        configuration_parameters,
+        modules_configuration_parameters_map,
     )
 
     logging.info(
@@ -38,7 +43,12 @@ def spin_up_local_redis(
 
 
 def generate_standalone_redis_server_args(
-    binary, dbdir, local_module_files, port, configuration_parameters=None
+    binary,
+    dbdir,
+    local_module_files,
+    port,
+    configuration_parameters=None,
+    modules_configuration_parameters_map={},
 ):
     # start redis-server
     command = [
@@ -60,18 +70,12 @@ def generate_standalone_redis_server_args(
             )
     if local_module_files is not None:
         if type(local_module_files) == str:
-            command.extend(
-                [
-                    "--loadmodule",
-                    os.path.abspath(local_module_files),
-                ]
+            redis_server_config_module_part(
+                command, local_module_files, modules_configuration_parameters_map
             )
         if type(local_module_files) == list:
             for mod in local_module_files:
-                command.extend(
-                    [
-                        "--loadmodule",
-                        os.path.abspath(mod),
-                    ]
+                redis_server_config_module_part(
+                    command, mod, modules_configuration_parameters_map
                 )
     return command

redisbench_admin/run/cluster.py

@@ -111,6 +111,7 @@ def spin_up_redis_cluster_remote_redis(
     shard_count,
     start_port,
     ssh_port,
+    modules_configuration_parameters_map,
 ):
     logging.info("Generating the remote redis-server command arguments")
     redis_process_commands = []
@@ -123,6 +124,7 @@ def spin_up_redis_cluster_remote_redis(
             server_private_ip,
             shard_port,
             redis_configuration_parameters,
+            modules_configuration_parameters_map,
         )
         logging.error(
             "Remote primary shard {} command: {}".format(

redisbench_admin/run/run.py

@@ -42,7 +42,7 @@ def define_benchmark_plan(benchmark_definitions, default_specs):
             benchmark_runs_plan[benchmark_type] = {}
 
         # extract dataset-name
-        dbconfig_present, dataset_name, _, _ = extract_redis_dbconfig_parameters(
+        dbconfig_present, dataset_name, _, _, _ = extract_redis_dbconfig_parameters(
             benchmark_config, "dbconfig"
         )
         if dataset_name is None:

redisbench_admin/run_local/local_db.py

@@ -68,6 +68,7 @@ def local_db_spin(
         _,
         redis_configuration_parameters,
         dataset_load_timeout_secs,
+        modules_configuration_parameters_map,
     ) = extract_redis_dbconfig_parameters(benchmark_config, "dbconfig")
     cluster_api_enabled = False
     logging.info(
@@ -85,6 +86,7 @@ def local_db_spin(
             local_module_file,
             redis_configuration_parameters,
             dataset_load_timeout_secs,
+            modules_configuration_parameters_map,
         )
 
         status = setup_redis_cluster_from_conns(
@@ -111,6 +113,7 @@ def local_db_spin(
             redis_configuration_parameters,
             dbdir_folder,
             dataset_load_timeout_secs,
+            modules_configuration_parameters_map,
         )
 
         r = redis.StrictRedis(port=args.port)

redisbench_admin/run_remote/remote_db.py

@@ -90,6 +90,7 @@ def remote_db_spin(
         _,
         redis_configuration_parameters,
         dataset_load_timeout_secs,
+        modules_configuration_parameters_map,
     ) = extract_redis_dbconfig_parameters(benchmark_config, "dbconfig")
 
     cluster_start_port = 20000
@@ -129,6 +130,7 @@ def remote_db_spin(
             shard_count,
             cluster_start_port,
             db_ssh_port,
+            modules_configuration_parameters_map,
         )
 
         for p in range(cluster_start_port, cluster_start_port + shard_count):
@@ -153,6 +155,7 @@ def remote_db_spin(
             logname,
             redis_configuration_parameters,
             db_ssh_port,
+            modules_configuration_parameters_map,
         )
         full_logfiles.append(full_logfile)
         local_redis_conn, ssh_tunnel = ssh_tunnel_redisconn(

redisbench_admin/run_remote/standalone.py

@@ -12,6 +12,7 @@
     execute_remote_commands,
 )
 from redisbench_admin.utils.ssh import SSHSession
+from redisbench_admin.utils.utils import redis_server_config_module_part
 
 
 def spin_up_standalone_remote_redis(
@@ -23,10 +24,15 @@ def spin_up_standalone_remote_redis(
     logfile,
     redis_configuration_parameters=None,
     port=22,
+    modules_configuration_parameters_map={},
 ):
 
     full_logfile, initial_redis_cmd = generate_remote_standalone_redis_cmd(
-        logfile, redis_configuration_parameters, remote_module_files, temporary_dir
+        logfile,
+        redis_configuration_parameters,
+        remote_module_files,
+        temporary_dir,
+        modules_configuration_parameters_map,
     )
 
     # start redis-server
@@ -84,7 +90,11 @@ def remote_module_files_cp(
 
 
 def generate_remote_standalone_redis_cmd(
-    logfile, redis_configuration_parameters, remote_module_files, temporary_dir
+    logfile,
+    redis_configuration_parameters,
+    remote_module_files,
+    temporary_dir,
+    modules_configuration_parameters_map,
 ):
     initial_redis_cmd = 'redis-server --save "" --logfile {} --dir {} --daemonize yes --protected-mode no'.format(
         logfile, temporary_dir
@@ -98,7 +108,17 @@ def generate_remote_standalone_redis_cmd(
             initial_redis_cmd += " --{} {}".format(
                 configuration_parameter, configuration_value
             )
+    command = []
+    if remote_module_files is not None:
+        if type(remote_module_files) == str:
+            redis_server_config_module_part(
+                command, remote_module_files, modules_configuration_parameters_map
+            )
+        if type(remote_module_files) == list:
+            for mod in remote_module_files:
+                redis_server_config_module_part(
+                    command, mod, modules_configuration_parameters_map
+                )
     if remote_module_files is not None:
-        for remote_module_file in remote_module_files:
-            initial_redis_cmd += " --loadmodule {}".format(remote_module_file)
+        initial_redis_cmd += " ".join(command)
     return full_logfile, initial_redis_cmd

redisbench_admin/utils/benchmark_config.py

@@ -173,13 +173,18 @@ def merge_default_and_specific_properties_dict_type(
 
 def extract_redis_dbconfig_parameters(benchmark_config, dbconfig_keyname):
     redis_configuration_parameters = {}
+    modules_configuration_parameters_map = {}
     dataset_load_timeout_secs = 120
     dataset_name = None
     dbconfig_present = False
     if dbconfig_keyname in benchmark_config:
         dbconfig_present = True
         if type(benchmark_config[dbconfig_keyname]) == list:
             for k in benchmark_config[dbconfig_keyname]:
+                if "module-configuration-parameters" in k:
+                    modules_configuration_parameters_map = k[
+                        "module-configuration-parameters"
+                    ]
                 if "configuration-parameters" in k:
                     cp = k["configuration-parameters"]
                     for item in cp:
@@ -190,6 +195,10 @@ def extract_redis_dbconfig_parameters(benchmark_config, dbconfig_keyname):
                 if "dataset_name" in k:
                     dataset_name = k["dataset_name"]
         if type(benchmark_config[dbconfig_keyname]) == dict:
+            if "module-configuration-parameters" in benchmark_config[dbconfig_keyname]:
+                modules_configuration_parameters_map = benchmark_config[
+                    dbconfig_keyname
+                ]["module-configuration-parameters"]
             if "configuration-parameters" in benchmark_config[dbconfig_keyname]:
                 cp = benchmark_config[dbconfig_keyname]["configuration-parameters"]
                 for k, v in cp.items():
@@ -204,6 +213,7 @@ def extract_redis_dbconfig_parameters(benchmark_config, dbconfig_keyname):
         dataset_name,
         redis_configuration_parameters,
         dataset_load_timeout_secs,
+        modules_configuration_parameters_map,
     )
 
 

redisbench_admin/utils/utils.py

@@ -25,6 +25,36 @@
 EPOCH = dt.datetime.utcfromtimestamp(0)
 
 
+def redis_server_config_module_part(
+    command, local_module_file, modules_configuration_parameters_map
+):
+    command.extend(
+        [
+            "--loadmodule",
+            os.path.abspath(local_module_file),
+        ]
+    )
+    for (
+        module_config_modulename,
+        module_config_dict,
+    ) in modules_configuration_parameters_map.items():
+        if module_config_modulename in local_module_file:
+            for (
+                module_config_parameter_name,
+                module_config_parameter_value,
+            ) in module_config_dict.items():
+                if type(module_config_parameter_value) != str:
+                    module_config_parameter_value = "{}".format(
+                        module_config_parameter_value
+                    )
+                command.extend(
+                    [
+                        module_config_parameter_name,
+                        module_config_parameter_value,
+                    ]
+                )
+
+
 def upload_artifacts_to_s3(
     artifacts,
     s3_bucket_name,