Fix redis-benchmark v6.2.4 WARNING:* parsing

pyproject.toml

@@ -28,6 +28,7 @@ PyYAML = "^5.4.0"
 wget = "^3.2"
 pytablewriter = "^0.60.0"
 sshtunnel = "^0.4.0"
+pyWorkFlow = "^0.0.2"
 
 [tool.poetry.dev-dependencies]
 pytest = "^4.6"

redisbench_admin/run/aibench_run_inference_redisai_vision/__init__.py

@@ -0,0 +1,5 @@
+#  Apache License Version 2.0
+#
+#  Copyright (c) 2021., Redis Labs Modules
+#  All rights reserved.
+#

redisbench_admin/run/aibench_run_inference_redisai_vision/aibench_run_inference_redisai_vision.py

@@ -0,0 +1,39 @@
+#  Apache License Version 2.0
+#
+#  Copyright (c) 2021., Redis Labs Modules
+#  All rights reserved.
+#
+
+from redisbench_admin.utils.local import check_if_needs_remote_fetch
+
+
+def prepare_aibench_benchmark_command(
+    executable_path: str,
+    server_private_ip: object,
+    server_plaintext_port: object,
+    benchmark_config: object,
+    current_workdir,
+    result_file: str,
+    remote_queries_file,
+    is_remote: bool,
+):
+    command_arr = [executable_path]
+
+    command_arr.extend(["--host", "{}".format(server_private_ip)])
+    command_arr.extend(["--port", "{}".format(server_plaintext_port)])
+    if "parameters" in benchmark_config:
+        for k in benchmark_config["parameters"]:
+            if "file" in k:
+                input_file = k["file"]
+                input_file = check_if_needs_remote_fetch(
+                    input_file, "/tmp", None, remote_queries_file, is_remote
+                )
+                command_arr.extend(["--file", input_file])
+            else:
+                for kk in k.keys():
+                    command_arr.extend(["--{}".format(kk), str(k[kk])])
+
+    command_arr.extend(["--json-out-file", result_file])
+
+    command_str = " ".join(command_arr)
+    return command_arr, command_str

redisbench_admin/run/common.py

@@ -10,6 +10,9 @@
 
 import redis
 
+from redisbench_admin.run.aibench_run_inference_redisai_vision.aibench_run_inference_redisai_vision import (
+    prepare_aibench_benchmark_command,
+)
 from redisbench_admin.run.redis_benchmark.redis_benchmark import (
     prepare_redis_benchmark_command,
 )
@@ -96,6 +99,21 @@ def prepare_benchmark_parameters(
                     input_data_file,
                     isremote,
                 )
+            if "aibench_" in benchmark_tool:
+                input_data_file = None
+                if isremote is True:
+                    benchmark_tool = "/tmp/{}".format(benchmark_tool)
+                    input_data_file = "/tmp/input.data"
+                (command_arr, command_str,) = prepare_aibench_benchmark_command(
+                    benchmark_tool,
+                    server_private_ip,
+                    server_plaintext_port,
+                    entry,
+                    current_workdir,
+                    remote_results_file,
+                    input_data_file,
+                    isremote,
+                )
     printed_command_str = command_str
     printed_command_arr = command_arr
     if len(command_str) > 200:

redisbench_admin/run/redis_benchmark/redis_benchmark.py

@@ -22,22 +22,25 @@ def redis_benchmark_from_stdout_csv_to_json(
         "StartTimeHuman": start_time_str,
     }
     csv_data = csv_data.splitlines()
-    full_csv = list(csv.reader(csv_data, delimiter=",", quoting=csv.QUOTE_ALL))
-    if len(full_csv) >= 2:
-        header = full_csv[0]
-        for raw_row in csv_data[1:]:
-            row = raw_row.rsplit(",", len(header) - 1)
-            assert len(row) == len(header)
-            test_name = row[0][1:-1].split(" ")[0]
-            if overload_test_name is not None:
-                test_name = overload_test_name
-            results_dict["Tests"][test_name] = {}
-            for pos, value in enumerate(row[1:]):
-                if '"' == value[0]:
-                    value = value[1:]
-                if '"' == value[-1]:
-                    value = value[:-1]
-                results_dict["Tests"][test_name][header[pos + 1]] = value
+    if len(csv_data) > 0:
+        if "WARNING:" in csv_data[0]:
+            csv_data = csv_data[1:]
+        full_csv = list(csv.reader(csv_data, delimiter=",", quoting=csv.QUOTE_ALL))
+        if len(full_csv) >= 2:
+            header = full_csv[0]
+            for raw_row in csv_data[1:]:
+                row = raw_row.rsplit(",", len(header) - 1)
+                assert len(row) == len(header)
+                test_name = row[0][1:-1].split(" ")[0]
+                if overload_test_name is not None:
+                    test_name = overload_test_name
+                results_dict["Tests"][test_name] = {}
+                for pos, value in enumerate(row[1:]):
+                    if '"' == value[0]:
+                        value = value[1:]
+                    if '"' == value[-1]:
+                        value = value[:-1]
+                    results_dict["Tests"][test_name][header[pos + 1]] = value
     return results_dict
 
 

redisbench_admin/run/tsbs_run_queries_redistimeseries/tsbs_run_queries_redistimeseries.py

@@ -3,8 +3,6 @@
 #  Copyright (c) 2021., Redis Labs Modules
 #  All rights reserved.
 #
-import csv
-import re
 
 from redisbench_admin.utils.local import check_if_needs_remote_fetch
 
@@ -49,30 +47,3 @@ def prepare_tsbs_benchmark_command(
 
     command_str = " ".join(command_arr)
     return command_arr, command_str
-
-
-def post_process_ycsb_results(stdout, start_time_ms, start_time_str):
-    results_dict = {
-        "Tests": {},
-        "StartTime": start_time_ms,
-        "StartTimeHuman": start_time_str,
-    }
-    if type(stdout) == bytes:
-        stdout = stdout.decode("ascii")
-    csv_data = list(csv.reader(stdout.splitlines(), delimiter=","))
-    start_row = 0
-    for row in csv_data:
-        if len(row) >= 1:
-            if "[OVERALL]" in row[0]:
-                break
-        start_row = start_row + 1
-    for row in csv_data[start_row:]:
-        if len(row) >= 3:
-            op_group = row[0].strip()[1:-1]
-            metric_name = row[1].strip()
-            metric_name = re.sub("[^0-9a-zA-Z]+", "_", metric_name)
-            value = row[2].strip()
-            if op_group not in results_dict["Tests"]:
-                results_dict["Tests"][op_group] = {}
-            results_dict["Tests"][op_group][metric_name] = value
-    return results_dict

redisbench_admin/run_local/args.py

@@ -19,11 +19,19 @@
 
 
 def create_run_local_arguments(parser):
-    parser.add_argument("--module_path", type=str, required=True)
+    parser.add_argument("--module_path", type=str, required=False)
+    parser.add_argument(
+        "--dbdir_folder",
+        type=str,
+        required=False,
+        help="If specified the entire contents of the folder are copied to the redis dir.",
+    )
     parser.add_argument(
         "--allowed-tools",
         type=str,
-        default="redis-benchmark,redisgraph-benchmark-go,ycsb,tsbs_run_queries_redistimeseries,tsbs_load_redistimeseries",
+        default="redis-benchmark,redisgraph-benchmark-go,ycsb,"
+        + "tsbs_run_queries_redistimeseries,tsbs_load_redistimeseries,"
+        + "aibench_run_inference_redisai_vision",
         help="comma separated list of allowed tools for this module. By default all the supported are allowed.",
     )
     parser.add_argument(

redisbench_admin/run_local/run_local.py

@@ -69,6 +69,7 @@ def run_local_command_logic(args):
     ) = extract_git_vars()
 
     local_module_file = args.module_path
+    dbdir_folder = args.dbdir_folder
     os.path.abspath(".")
     required_modules = args.required_module
     profilers_enabled = args.enable_profilers
@@ -145,6 +146,16 @@ def run_local_command_logic(args):
                         temporary_dir
                     )
                 )
+                if dbdir_folder is not None:
+                    from distutils.dir_util import copy_tree
+
+                    copy_tree(dbdir_folder, temporary_dir)
+                    logging.info(
+                        "Copied entire content of {} into temporary path: {}".format(
+                            dbdir_folder, temporary_dir
+                        )
+                    )
+
                 check_dataset_local_requirements(
                     benchmark_config, temporary_dir, dirname
                 )
@@ -158,6 +169,7 @@ def run_local_command_logic(args):
                     args.port,
                     local_module_file,
                     redis_configuration_parameters,
+                    dbdir_folder,
                 )
 
                 if is_process_alive(redis_process) is False:
@@ -305,6 +317,7 @@ def run_local_command_logic(args):
                                 artifact_name,
                                 profile_artifact,
                             ) in profile_res_artifacts_map.items():
+                                s3_link = None
                                 if args.upload_results_s3:
                                     logging.info(
                                         "Uploading results to s3. s3 bucket name: {}. s3 bucket path: {}".format(
@@ -316,7 +329,7 @@ def run_local_command_logic(args):
                                         s3_bucket_name,
                                         s3_bucket_path,
                                     )
-                                s3_link = list(url_map.values())[0]
+                                    s3_link = list(url_map.values())[0]
                                 profilers_artifacts_matrix.append(
                                     [
                                         test_name,

redisbench_admin/run_remote/args.py

@@ -21,11 +21,19 @@
 
 
 def create_run_remote_arguments(parser):
-    parser.add_argument("--module_path", type=str, required=True)
+    parser.add_argument("--module_path", type=str, required=False)
+    parser.add_argument(
+        "--dbdir_folder",
+        type=str,
+        required=False,
+        help="If specified the entire contents of the folder are copied to the redis dir.",
+    )
     parser.add_argument(
         "--allowed-tools",
         type=str,
-        default="redis-benchmark,redisgraph-benchmark-go,ycsb,tsbs_run_queries_redistimeseries,tsbs_load_redistimeseries",
+        default="redis-benchmark,redisgraph-benchmark-go,ycsb,"
+        + "tsbs_run_queries_redistimeseries,tsbs_load_redistimeseries,"
+        + "aibench_run_inference_redisai_vision",
         help="comma separated list of allowed tools for this module. By default all the supported are allowed.",
     )
     parser.add_argument(

redisbench_admin/run_remote/run_remote.py

@@ -220,6 +220,7 @@ def run_remote_command_logic(args):
     tf_setup_name_sufix = "{}-{}".format(args.setup_name_sufix, tf_github_sha)
     s3_bucket_name = args.s3_bucket_name
     local_module_file = args.module_path
+    dbdir_folder = args.dbdir_folder
 
     if args.skip_env_vars_verify is False:
         check_ec2_env()
@@ -377,6 +378,7 @@ def run_remote_command_logic(args):
                         remote_dataset_file,
                         dirname,
                         redis_configuration_parameters,
+                        dbdir_folder,
                     )
                     dataset_load_start_time = datetime.datetime.now()
                     local_redis_conn, ssh_tunnel = ssh_tunnel_redisconn(
@@ -459,6 +461,23 @@ def run_remote_command_logic(args):
                             tool_link,
                         ) = extract_tsbs_extra_links(benchmark_config, benchmark_tool)
 
+                        setup_remote_benchmark_tool_requirements_tsbs(
+                            client_public_ip,
+                            username,
+                            private_key,
+                            tool_link,
+                            queries_file_link,
+                            remote_tool_link,
+                        )
+                    if "aibench_" in benchmark_tool:
+                        (
+                            queries_file_link,
+                            remote_tool_link,
+                            tool_link,
+                        ) = extract_aibench_extra_links(
+                            benchmark_config, benchmark_tool
+                        )
+
                         setup_remote_benchmark_tool_requirements_tsbs(
                             client_public_ip,
                             username,
@@ -867,6 +886,21 @@ def extract_tsbs_extra_links(benchmark_config, benchmark_tool):
     return queries_file_link, remote_tool_link, tool_link
 
 
+def extract_aibench_extra_links(benchmark_config, benchmark_tool):
+    remote_tool_link = "/tmp/{}".format(benchmark_tool)
+    tool_link = (
+        "https://s3.amazonaws.com/benchmarks.redislabs/"
+        + "tools/redisai/aibench/{}_linux_amd64".format(benchmark_tool)
+    )
+    queries_file_link = None
+    for entry in benchmark_config["clientconfig"]:
+        if "parameters" in entry:
+            for parameter in entry["parameters"]:
+                if "file" in parameter:
+                    queries_file_link = parameter["file"]
+    return queries_file_link, remote_tool_link, tool_link
+
+
 def get_test_s3_bucket_path(
     s3_bucket_name, test_name, tf_github_org, tf_github_repo, folder="results"
 ):

redisbench_admin/utils/local.py

@@ -70,10 +70,7 @@ def check_if_needs_remote_fetch(
 
 
 def spin_up_local_redis(
-    dbdir,
-    port,
-    local_module_file,
-    configuration_parameters=None,
+    dbdir, port, local_module_file, configuration_parameters=None, dbdir_folder=None
 ):
     command = generate_standalone_redis_server_args(
         dbdir, local_module_file, port, configuration_parameters